From ad6a0e9c31e077f76f22b3635ce240df54cc8493 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 13:38:02 +0530 Subject: [PATCH 01/20] linprog --- web/packages/new/photos/pages/duplicates.tsx | 31 ++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/web/packages/new/photos/pages/duplicates.tsx b/web/packages/new/photos/pages/duplicates.tsx index c8651198f7..59dce639c8 100644 --- a/web/packages/new/photos/pages/duplicates.tsx +++ b/web/packages/new/photos/pages/duplicates.tsx @@ -19,13 +19,14 @@ import SortIcon from "@mui/icons-material/Sort"; import { Box, Checkbox, - CircularProgress, Divider, IconButton, + LinearProgress, Stack, styled, Tooltip, Typography, + type LinearProgressProps, } from "@mui/material"; import { useRouter } from "next/router"; import React, { @@ -641,11 +642,18 @@ const DeduplicateButton: React.FC = ({ disabled={prunableCount == 0 || isDeduping} onClick={onRemoveDuplicates} > - + {isDeduping ? ( - - - + ) : ( <> @@ -659,3 +667,16 @@ const DeduplicateButton: React.FC = ({ ); + +interface LinearProgressWithLabelProps { + value: Exclude; +} + +export const LinearProgressWithLabel: React.FC< + LinearProgressWithLabelProps +> = ({ value }) => ( + + + `{Math.round(value)}% + +); From 3919fb0db2d1428ac3a5dcb01cb182b3b6c0d151 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 14:42:05 +0530 Subject: [PATCH 02/20] Progress is not tied to specific groups --- web/packages/new/photos/pages/duplicates.tsx | 86 ++++++++++---------- web/packages/new/photos/services/dedup.ts | 23 ++---- 2 files changed, 49 insertions(+), 60 deletions(-) diff --git a/web/packages/new/photos/pages/duplicates.tsx b/web/packages/new/photos/pages/duplicates.tsx index 59dce639c8..e0fef305c0 100644 --- a/web/packages/new/photos/pages/duplicates.tsx +++ b/web/packages/new/photos/pages/duplicates.tsx @@ -8,7 +8,6 @@ import { OverflowMenuOption, } from "@/base/components/OverflowMenu"; import { Ellipsized2LineTypography } from "@/base/components/Typography"; -import { errorDialogAttributes } from "@/base/components/utils/dialog"; import { pt } from "@/base/i18n"; import log from "@/base/log"; import { formattedByteSize } from "@/new/photos/utils/units"; @@ -59,7 +58,7 @@ import { import { useAppContext } from "../types/context"; const Page: React.FC = () => { - const { showMiniDialog } = useAppContext(); + const { onGenericError } = useAppContext(); const [state, dispatch] = useReducer(dedupReducer, initialDedupState); @@ -81,18 +80,18 @@ const Page: React.FC = () => { dispatch({ type: "dedupe" }); void removeSelectedDuplicateGroups( state.duplicateGroups, - (duplicateGroup: DuplicateGroup) => - dispatch({ type: "didRemoveDuplicateGroup", duplicateGroup }), - ).then((allSuccess) => { - dispatch({ type: "dedupeCompleted" }); - if (!allSuccess) { - const msg = pt( - "Some errors occurred when trying to remove duplicates.", - ); - showMiniDialog(errorDialogAttributes(msg)); - } - }); - }, [state.duplicateGroups, showMiniDialog]); + (progress: number) => + dispatch({ type: "setDedupeProgress", progress }), + ) + .then((removedGroupIDs) => + dispatch({ type: "dedupeCompleted", removedGroupIDs }), + ) + + .catch((e: unknown) => { + onGenericError(e); + dispatch({ type: "dedupeFailed" }); + }); + }, [state.duplicateGroups, onGenericError]); const contents = (() => { switch (state.status) { @@ -114,7 +113,7 @@ const Page: React.FC = () => { } prunableCount={state.prunableCount} prunableSize={state.prunableSize} - isDeduping={state.isDeduping} + dedupeProgress={state.dedupeProgress} onRemoveDuplicates={handleRemoveDuplicates} /> ); @@ -145,8 +144,6 @@ type SortOrder = "prunableCount" | "prunableSize"; interface DedupState { /** Status of the screen, between initial state => analysis */ status: undefined | "analyzing" | "analysisFailed" | "analysisCompleted"; - /** `true` if a dedupe is in progress. */ - isDeduping: boolean; /** * Groups of duplicates. * @@ -172,6 +169,11 @@ interface DedupState { * current selection. */ prunableSize: number; + /** + * If a dedupe is in progress, then this will indicate its progress + * percentage (a number between 0 and 100). + */ + dedupeProgress: number | undefined; } type DedupAction = @@ -182,16 +184,17 @@ type DedupAction = | { type: "toggleSelection"; index: number } | { type: "deselectAll" } | { type: "dedupe" } - | { type: "didRemoveDuplicateGroup"; duplicateGroup: DuplicateGroup } - | { type: "dedupeCompleted" }; + | { type: "setDedupeProgress"; progress: number } + | { type: "dedupeFailed" } + | { type: "dedupeCompleted"; removedGroupIDs: Set }; const initialDedupState: DedupState = { status: undefined, - isDeduping: false, duplicateGroups: [], sortOrder: "prunableSize", prunableCount: 0, prunableSize: 0, + dedupeProgress: undefined, }; const dedupReducer: React.Reducer = ( @@ -263,11 +266,18 @@ const dedupReducer: React.Reducer = ( } case "dedupe": - return { ...state, isDeduping: true }; + return { ...state, dedupeProgress: 0 }; - case "didRemoveDuplicateGroup": { + case "setDedupeProgress": { + return { ...state, dedupeProgress: action.progress }; + } + + case "dedupeFailed": + return { ...state, dedupeProgress: undefined }; + + case "dedupeCompleted": { const duplicateGroups = state.duplicateGroups.filter( - ({ id }) => id != action.duplicateGroup.id, + ({ id }) => !action.removedGroupIDs.has(id), ); const { prunableCount, prunableSize } = deducePrunableCountAndSize(duplicateGroups); @@ -278,9 +288,6 @@ const dedupReducer: React.Reducer = ( prunableSize, }; } - - case "dedupeCompleted": - return { ...state, isDeduping: false }; } }; @@ -612,34 +619,25 @@ const ItemGrid = styled("div", { `, ); -interface DeduplicateButtonProps { - /** - * See {@link prunableCount} in {@link DedupState}. - */ - prunableCount: number; - /** - * See {@link prunableSize} in {@link DedupState}. - */ - prunableSize: number; - /** - * `true` if a deduplication is in progress - */ - isDeduping: DedupState["isDeduping"]; +type DeduplicateButtonProps = Pick< + DedupState, + "prunableCount" | "prunableSize" | "dedupeProgress" +> & { /** * Called when the user presses the button to remove duplicates. */ onRemoveDuplicates: () => void; -} +}; const DeduplicateButton: React.FC = ({ prunableCount, prunableSize, - isDeduping, + dedupeProgress, onRemoveDuplicates, }) => ( = ({ flex: 1, }} > - {isDeduping ? ( - + {dedupeProgress !== undefined ? ( + ) : ( <> diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index 108ad3e131..a7d8b85754 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -1,7 +1,6 @@ import { assertionFailed } from "@/base/assert"; import { newID } from "@/base/id"; import { ensureLocalUser } from "@/base/local-user"; -import log from "@/base/log"; import type { EnteFile } from "@/media/file"; import { metadataHash } from "@/media/file-metadata"; import { wait } from "@/utils/promise"; @@ -213,29 +212,21 @@ export const deduceDuplicates = async () => { * * This function will only process entries for which isSelected is `true`. * - * @param onRemoveDuplicateGroup A function that is called each time a duplicate - * group is successfully removed. The duplicate group that was removed is passed - * as an argument to it. + * @param onProgress A function that is called with an estimated progress + * percentage of the operation (a number between 0 and 100). * - * @returns true if all selected duplicate groups were successfully removed, and - * false if there were any errors. + * @returns A set containing the IDs of the duplicate groups that were removed. */ export const removeSelectedDuplicateGroups = async ( duplicateGroups: DuplicateGroup[], - onRemoveDuplicateGroup: (g: DuplicateGroup) => void, + onProgress: (progress: number) => void, ) => { const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected); - let allSuccess = true; for (const duplicateGroup of selectedDuplicateGroups) { - try { - await removeDuplicateGroup(duplicateGroup); - onRemoveDuplicateGroup(duplicateGroup); - } catch (e) { - log.warn("Failed to remove duplicate group", e); - allSuccess = false; - } + await removeDuplicateGroup(duplicateGroup); + console.log(onProgress); } - return allSuccess; + return new Set(selectedDuplicateGroups.map((g) => g.id)); }; /** From 7348170a366a29d2695ef6c65ab3c186d3295e65 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 14:57:46 +0530 Subject: [PATCH 03/20] Tweak progress --- web/packages/new/photos/pages/duplicates.tsx | 23 +++++++------------- web/packages/new/photos/services/dedup.ts | 6 +++++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/web/packages/new/photos/pages/duplicates.tsx b/web/packages/new/photos/pages/duplicates.tsx index e0fef305c0..5802ad6b95 100644 --- a/web/packages/new/photos/pages/duplicates.tsx +++ b/web/packages/new/photos/pages/duplicates.tsx @@ -25,7 +25,6 @@ import { styled, Tooltip, Typography, - type LinearProgressProps, } from "@mui/material"; import { useRouter } from "next/router"; import React, { @@ -286,6 +285,7 @@ const dedupReducer: React.Reducer = ( duplicateGroups, prunableCount, prunableSize, + dedupeProgress: undefined, }; } } @@ -651,7 +651,13 @@ const DeduplicateButton: React.FC = ({ }} > {dedupeProgress !== undefined ? ( - + ) : ( <> @@ -665,16 +671,3 @@ const DeduplicateButton: React.FC = ({ ); - -interface LinearProgressWithLabelProps { - value: Exclude; -} - -export const LinearProgressWithLabel: React.FC< - LinearProgressWithLabelProps -> = ({ value }) => ( - - - `{Math.round(value)}% - -); diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index a7d8b85754..4304f9e749 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -222,9 +222,15 @@ export const removeSelectedDuplicateGroups = async ( onProgress: (progress: number) => void, ) => { const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected); + let i = selectedDuplicateGroups.length; for (const duplicateGroup of selectedDuplicateGroups) { await removeDuplicateGroup(duplicateGroup); console.log(onProgress); + onProgress( + ((selectedDuplicateGroups.length - i++) / + selectedDuplicateGroups.length) * + -100, + ); } return new Set(selectedDuplicateGroups.map((g) => g.id)); }; From 3f21011392d9389c318fe5737a7ebdf8a07884ee Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 15:19:15 +0530 Subject: [PATCH 04/20] Retain files so that we can reuse trashFiles code --- web/packages/new/photos/services/dedup.ts | 63 +++++++++++++---------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index 4304f9e749..c2e3c08c85 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -26,16 +26,24 @@ export interface DuplicateGroup { */ items: { /** - * The underlying collection file. + * The underlying file to delete. + * + * This is one of the files from amongst {@link collectionFiles}, + * arbitrarily picked to stand in for the entire set of files in the UI. */ file: EnteFile; /** - * The IDs of the collections to which this file belongs. + * All the collection files for the underlying file. + * + * This includes {@link file} too. */ - collectionIDs: number[]; + collectionFiles: EnteFile[]; /** - * The name of the collection (or of one of them, arbitrarily picked) to - * which this file belongs. + * The name of the collection to which {@link file} belongs. + * + * Like {@link file} itself, this is an arbitrary pick. Logically, none + * of the collections to which the file belongs are given more + * preference than the other. */ collectionName: string; }[]; @@ -112,9 +120,9 @@ export const deduceDuplicates = async () => { ); // Group the filtered collection files by their hashes, keeping only one - // entry per file ID, but also retaining all the collections IDs to which - // that file belongs. - const collectionIDsByFileID = new Map(); + // entry per file ID. We also retain all the collections files for a + // particular file ID. + const collectionFilesByFileID = new Map(); const filesByHash = new Map(); for (const file of filteredCollectionFiles) { const hash = metadataHash(file.metadata); @@ -124,15 +132,15 @@ export const deduceDuplicates = async () => { continue; } - const collectionIDs = collectionIDsByFileID.get(file.id); - if (!collectionIDs) { + const collectionFiles = collectionFilesByFileID.get(file.id); + if (!collectionFiles) { // This is the first collection file we're seeing for a particular // file ID, so also create an entry in the filesByHash map. filesByHash.set(hash, [...(filesByHash.get(hash) ?? []), file]); } - collectionIDsByFileID.set(file.id, [ - ...(collectionIDs ?? []), - file.collectionID, + collectionFilesByFileID.set(file.id, [ + ...(collectionFiles ?? []), + file, ]); } @@ -171,15 +179,15 @@ export const deduceDuplicates = async () => { const collectionName = collectionNameByID.get( file.collectionID, ); - const collectionIDs = collectionIDsByFileID.get(file.id); + const collectionFiles = collectionFilesByFileID.get(file.id); // Ignore duplicates for which we do not have a collection. This // shouldn't really happen though, so retain an assert. - if (!collectionName || !collectionIDs) { + if (!collectionName || !collectionFiles) { assertionFailed(); return undefined; } - return { file, collectionIDs, collectionName }; + return { file, collectionFiles, collectionName }; }) .filter((item) => !!item); if (items.length < 2) continue; @@ -222,15 +230,22 @@ export const removeSelectedDuplicateGroups = async ( onProgress: (progress: number) => void, ) => { const selectedDuplicateGroups = duplicateGroups.filter((g) => g.isSelected); - let i = selectedDuplicateGroups.length; + + // See: "Pruning duplicates" under [Note: Deduplication logic]. A tl;dr; is + // + // 1. For each selected duplicate group, determine the file to retain. + // 2. Add these to the user owned collections the other files exist in. + // 3. Delete the other files. + // + for (const duplicateGroup of selectedDuplicateGroups) { await removeDuplicateGroup(duplicateGroup); console.log(onProgress); - onProgress( - ((selectedDuplicateGroups.length - i++) / - selectedDuplicateGroups.length) * - -100, - ); + // onProgress( + // ((selectedDuplicateGroups.length - i++) / + // selectedDuplicateGroups.length) * + // -100, + // ); } return new Set(selectedDuplicateGroups.map((g) => g.id)); }; @@ -239,10 +254,6 @@ export const removeSelectedDuplicateGroups = async ( * Retain only file from amongst these duplicates whilst keeping the existing * collection entries intact. * - * See: "Pruning duplicates" under [Note: Deduplication logic]. To summarize: - * 1. Find the file to retain. - * 2. Add it to the user owned collections the other files exist in. - * 3. Delete the other files. */ const removeDuplicateGroup = async (duplicateGroup: DuplicateGroup) => { const fileToRetain = duplicateGroupFileToRetain(duplicateGroup); From 26fb47c1655dc18ec0c9d1a885ffe32f26a07492 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 15:50:35 +0530 Subject: [PATCH 05/20] Invert the processing --- web/packages/new/photos/services/dedup.ts | 70 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index c2e3c08c85..754194b69f 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -5,7 +5,7 @@ import type { EnteFile } from "@/media/file"; import { metadataHash } from "@/media/file-metadata"; import { wait } from "@/utils/promise"; import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; -import { createCollectionNameByID } from "./collection"; +import { addToCollection, createCollectionNameByID } from "./collection"; import { getLocalCollections } from "./collections"; import { getLocalFiles } from "./files"; @@ -238,9 +238,50 @@ export const removeSelectedDuplicateGroups = async ( // 3. Delete the other files. // + const filesToAdd = new Map(); + let filesToTrash: EnteFile[] = []; + for (const duplicateGroup of selectedDuplicateGroups) { - await removeDuplicateGroup(duplicateGroup); - console.log(onProgress); + const retainedItem = duplicateGroupItemToRetain(duplicateGroup); + // Find the existing collection IDs to which this item already belongs. + const existingCollectionIDs = new Set( + retainedItem.collectionFiles.map((cf) => cf.collectionID), + ); + // For each item, + for (const item of duplicateGroup.items) { + // except the one we're retaining, + if (item.file.id == retainedItem.file.id) continue; + // Add the file we're retaining to each collection to which this + // item belongs. + for (const { collectionID } of item.collectionFiles) { + // Skip if already there + if (existingCollectionIDs.has(collectionID)) continue; + filesToAdd.set(collectionID, [ + ...(filesToAdd.get(collectionID) ?? []), + retainedItem.file, + ]); + } + // Add it to the list of items to be trashed. + filesToTrash = filesToTrash.concat(item.collectionFiles); + } + } + + let np = 0; + const ntotal = filesToAdd.size + filesToTrash.length ? 1 : 0; + const tickProgress = () => onProgress((np++ / ntotal) * 100); + + // Process the adds. + const collections = await getLocalCollections("normal"); + const collectionsByID = new Map(collections.map((c) => [c.id, c])); + for (const [collectionID, collectionFiles] of filesToAdd.entries()) { + await addToCollection(collectionsByID.get(collectionID)!, collectionFiles) + tickProgress(); + } + + // Process the removes. + if (filesToTrash.length) { + // await trashFiles(filesToTrash); + // onProgress( // ((selectedDuplicateGroups.length - i++) / // selectedDuplicateGroups.length) * @@ -256,7 +297,6 @@ export const removeSelectedDuplicateGroups = async ( * */ const removeDuplicateGroup = async (duplicateGroup: DuplicateGroup) => { - const fileToRetain = duplicateGroupFileToRetain(duplicateGroup); console.log({ fileToRetain }); // const collections; @@ -265,26 +305,26 @@ const removeDuplicateGroup = async (duplicateGroup: DuplicateGroup) => { }; /** - * Find the most eligible file from amongst the duplicates to retain. + * Find the most eligible item from amongst the duplicates to retain. * * Give preference to files which have a caption or edited name or edited time, * otherwise pick arbitrarily. */ -const duplicateGroupFileToRetain = (duplicateGroup: DuplicateGroup) => { - const filesWithCaption: EnteFile[] = []; - const filesWithOtherEdits: EnteFile[] = []; - for (const { file } of duplicateGroup.items) { - const pubMM = getPublicMagicMetadataSync(file); +const duplicateGroupItemToRetain = (duplicateGroup: DuplicateGroup) => { + const itemsWithCaption: DuplicateGroup["items"] = []; + const itemsWithOtherEdits: DuplicateGroup["items"] = []; + for (const item of duplicateGroup.items) { + const pubMM = getPublicMagicMetadataSync(item.file); if (!pubMM) continue; - if (pubMM.caption) filesWithCaption.push(file); + if (pubMM.caption) itemsWithCaption.push(item); if (pubMM.editedName ?? pubMM.editedTime) - filesWithOtherEdits.push(file); + itemsWithOtherEdits.push(item); } // Duplicate group items should not be empty, so we'll get something always. return ( - filesWithCaption[0] ?? - filesWithOtherEdits[0] ?? - duplicateGroup.items[0]!.file + itemsWithCaption[0] ?? + itemsWithOtherEdits[0] ?? + duplicateGroup.items[0]! ); }; From 1b863005eae9e3461c5af2dbe19827436fd9aa53 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:03:09 +0530 Subject: [PATCH 06/20] Ref --- .../src/components/PhotoViewer/index.tsx | 4 +- web/apps/photos/src/pages/deduplicate.tsx | 6 +- .../photos/src/services/collectionService.ts | 2 +- web/apps/photos/src/services/fileService.ts | 56 ------------------- web/apps/photos/src/utils/file/index.ts | 5 +- web/packages/media/file.ts | 9 --- .../new/photos/services/collection/index.ts | 41 ++++++++++++++ 7 files changed, 49 insertions(+), 74 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/index.tsx b/web/apps/photos/src/components/PhotoViewer/index.tsx index 86dbf0195c..6b94315460 100644 --- a/web/apps/photos/src/components/PhotoViewer/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/index.tsx @@ -12,6 +12,7 @@ import { import { fileLogID, type EnteFile } from "@/media/file"; import { FileType } from "@/media/file-type"; import { isHEICExtension, needsJPEGConversion } from "@/media/formats"; +import { moveToTrash } from "@/new/photos/services/collection"; import { extractRawExif, parseExif } from "@/new/photos/services/exif"; import { AppContext } from "@/new/photos/types/context"; import { FlexWrapper } from "@ente/shared/components/Container"; @@ -58,7 +59,6 @@ import { addToFavorites, removeFromFavorites, } from "services/collectionService"; -import { trashFiles } from "services/fileService"; import { SetFilesDownloadProgressAttributesCreator } from "types/gallery"; import { copyFileToClipboard, @@ -550,7 +550,7 @@ function PhotoViewer(props: PhotoViewerProps) { try { showLoadingBar(); try { - await trashFiles([file]); + await moveToTrash([file]); } finally { hideLoadingBar(); } diff --git a/web/apps/photos/src/pages/deduplicate.tsx b/web/apps/photos/src/pages/deduplicate.tsx index a805151a7a..3b7b41bff6 100644 --- a/web/apps/photos/src/pages/deduplicate.tsx +++ b/web/apps/photos/src/pages/deduplicate.tsx @@ -2,7 +2,7 @@ import { stashRedirect } from "@/accounts/services/redirect"; import { ActivityIndicator } from "@/base/components/mui/ActivityIndicator"; import { errorDialogAttributes } from "@/base/components/utils/dialog"; import log from "@/base/log"; -import { ALL_SECTION } from "@/new/photos/services/collection"; +import { ALL_SECTION, moveToTrash } from "@/new/photos/services/collection"; import { getLocalCollections } from "@/new/photos/services/collections"; import { createFileCollectionIDs, @@ -24,7 +24,7 @@ import { default as Router, default as router } from "next/router"; import { createContext, useEffect, useState } from "react"; import { getAllLatestCollections } from "services/collectionService"; import { Duplicate, getDuplicates } from "services/deduplicationService"; -import { syncFiles, trashFiles } from "services/fileService"; +import { syncFiles } from "services/fileService"; import { syncTrash } from "services/trashService"; import { SelectedState } from "types/gallery"; import { getSelectedFiles } from "utils/file"; @@ -133,7 +133,7 @@ export default function Deduplicate() { try { showLoadingBar(); const selectedFiles = getSelectedFiles(selected, duplicateFiles); - await trashFiles(selectedFiles); + await moveToTrash(selectedFiles); // trashFiles above does an API request, we still need to update our // local state. diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index 81864d4c95..a26c684d30 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -63,7 +63,7 @@ const UNCATEGORIZED_COLLECTION_NAME = "Uncategorized"; export const HIDDEN_COLLECTION_NAME = ".hidden"; const FAVORITE_COLLECTION_NAME = "Favorites"; -export const REQUEST_BATCH_SIZE = 1000; +const REQUEST_BATCH_SIZE = 1000; export const getCollectionLastSyncTime = async (collection: Collection) => (await localForage.getItem(`${collection.id}-time`)) ?? 0; diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index a450a3bcba..ffde9fd9a1 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -8,7 +8,6 @@ import { EnteFile, FileWithUpdatedMagicMetadata, FileWithUpdatedPublicMagicMetadata, - TrashRequest, } from "@/media/file"; import { clearCachedThumbnailsIfChanged, @@ -16,14 +15,12 @@ import { getLocalFiles, setLocalFiles, } from "@/new/photos/services/files"; -import { batch } from "@/utils/array"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import exportService from "services/export"; import { decryptFile } from "utils/file"; import { getCollectionLastSyncTime, - REQUEST_BATCH_SIZE, setCollectionLastSyncTime, } from "./collectionService"; @@ -133,59 +130,6 @@ const removeDeletedCollectionFiles = async ( return files; }; -export const trashFiles = async (filesToTrash: EnteFile[]) => { - try { - const token = getToken(); - if (!token) { - return; - } - const batchedFilesToTrash = batch(filesToTrash, REQUEST_BATCH_SIZE); - for (const batch of batchedFilesToTrash) { - const trashRequest: TrashRequest = { - items: batch.map((file) => ({ - fileID: file.id, - collectionID: file.collectionID, - })), - }; - await HTTPService.post( - await apiURL("/files/trash"), - trashRequest, - null, - { - "X-Auth-Token": token, - }, - ); - } - } catch (e) { - log.error("trash file failed", e); - throw e; - } -}; - -export const deleteFromTrash = async (filesToDelete: number[]) => { - try { - const token = getToken(); - if (!token) { - return; - } - const batchedFilesToDelete = batch(filesToDelete, REQUEST_BATCH_SIZE); - - for (const batch of batchedFilesToDelete) { - await HTTPService.post( - await apiURL("/trash/delete"), - { fileIDs: batch }, - null, - { - "X-Auth-Token": token, - }, - ); - } - } catch (e) { - log.error("deleteFromTrash failed", e); - throw e; - } -}; - export interface UpdateMagicMetadataRequest { id: number; magicMetadata: EncryptedMagicMetadata; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 5223a38e26..85643640c7 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -19,6 +19,7 @@ import { import { ItemVisibility } from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; +import { deleteFromTrash, moveToTrash } from "@/new/photos/services/collection"; import { isArchivedFile, updateMagicMetadata, @@ -33,8 +34,6 @@ import { moveToHiddenCollection, } from "services/collectionService"; import { - deleteFromTrash, - trashFiles, updateFileMagicMetadata, updateFilePublicMagicMetadata, } from "services/fileService"; @@ -544,7 +543,7 @@ export const handleFileOps = async ( case FILE_OPS_TYPE.TRASH: try { markTempDeleted(files); - await trashFiles(files); + await moveToTrash(files); } catch (e) { clearTempDeleted(); throw e; diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts index 92dee2b6e1..1082ca5280 100644 --- a/web/packages/media/file.ts +++ b/web/packages/media/file.ts @@ -182,15 +182,6 @@ export interface EnteFile key: string; } -export interface TrashRequest { - items: TrashRequestItems[]; -} - -export interface TrashRequestItems { - fileID: number; - collectionID: number; -} - export interface FileWithUpdatedMagicMetadata { file: EnteFile; updatedMagicMetadata: FileMagicMetadata; diff --git a/web/packages/new/photos/services/collection/index.ts b/web/packages/new/photos/services/collection/index.ts index dd7c374155..33e602536c 100644 --- a/web/packages/new/photos/services/collection/index.ts +++ b/web/packages/new/photos/services/collection/index.ts @@ -217,3 +217,44 @@ const encryptWithCollectionKey = async ( }; }), ); + +/** + * Make a remote request to move the given {@link collectionFiles} to trash. + * + * Does not modify local state. + */ +export const moveToTrash = async (collectionFiles: EnteFile[]) => { + for (const batchFiles of batch(collectionFiles, requestBatchSize)) { + ensureOk( + await fetch(await apiURL("/files/trash"), { + method: "POST", + headers: await authenticatedRequestHeaders(), + body: JSON.stringify({ + items: batchFiles.map((file) => ({ + fileID: file.id, + collectionID: file.collectionID, + })), + }), + }), + ); + } +}; + +/** + * Make a remote request to delete the given {@link fileIDs} from trash. + * + * Does not modify local state. + */ +export const deleteFromTrash = async (fileIDs: number[]) => { + for (const batchIDs of batch(fileIDs, requestBatchSize)) { + ensureOk( + await fetch(await apiURL("/trash/delete"), { + method: "POST", + headers: await authenticatedRequestHeaders(), + body: JSON.stringify({ + fileIDs: batchIDs, + }), + }), + ); + } +}; From 7cd1ce0a99d3115e9a5551e1e15a5c440626a2b7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:13:59 +0530 Subject: [PATCH 07/20] Avoid the same name --- web/apps/photos/src/pages/deduplicate.tsx | 4 ++-- web/apps/photos/src/services/export/index.ts | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/pages/deduplicate.tsx b/web/apps/photos/src/pages/deduplicate.tsx index 3b7b41bff6..4b1d16630a 100644 --- a/web/apps/photos/src/pages/deduplicate.tsx +++ b/web/apps/photos/src/pages/deduplicate.tsx @@ -135,8 +135,8 @@ export default function Deduplicate() { const selectedFiles = getSelectedFiles(selected, duplicateFiles); await moveToTrash(selectedFiles); - // trashFiles above does an API request, we still need to update our - // local state. + // moveToTrash above does an API request, we still need to update + // our local state. // // Enhancement: This can be done in a more granular manner. Also, it // is better to funnel these syncs instead of adding these here and diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index c242ae1bca..48f76c8566 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -750,19 +750,19 @@ class ExportService { const { image, video } = parseLivePhotoExportName(fileExportName); - await moveToTrash( + await moveToFSTrash( exportDir, collectionExportName, image, ); - await moveToTrash( + await moveToFSTrash( exportDir, collectionExportName, video, ); } else { - await moveToTrash( + await moveToFSTrash( exportDir, collectionExportName, fileExportName, @@ -1459,14 +1459,15 @@ const isExportInProgress = (exportStage: ExportStage) => exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; /** - * Move {@link fileName} in {@link collectionName} to Trash. + * Move {@link fileName} in {@link collectionName} to the special per-collection + * file system "Trash" folder we created under the export directory. * * Also move its associated metadata JSON to Trash. * * @param exportDir The root directory on the user's file system where we are * exporting to. * */ -const moveToTrash = async ( +const moveToFSTrash = async ( exportDir: string, collectionName: string, fileName: string, From f5a3b8a3fb3b01d953dd6a6310a45f027a534b96 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:29:13 +0530 Subject: [PATCH 08/20] Del --- web/packages/new/photos/services/dedup.ts | 35 ++++++++--------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index 754194b69f..e8942146be 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -3,9 +3,12 @@ import { newID } from "@/base/id"; import { ensureLocalUser } from "@/base/local-user"; import type { EnteFile } from "@/media/file"; import { metadataHash } from "@/media/file-metadata"; -import { wait } from "@/utils/promise"; import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; -import { addToCollection, createCollectionNameByID } from "./collection"; +import { + addToCollection, + createCollectionNameByID, + moveToTrash, +} from "./collection"; import { getLocalCollections } from "./collections"; import { getLocalFiles } from "./files"; @@ -274,36 +277,22 @@ export const removeSelectedDuplicateGroups = async ( const collections = await getLocalCollections("normal"); const collectionsByID = new Map(collections.map((c) => [c.id, c])); for (const [collectionID, collectionFiles] of filesToAdd.entries()) { - await addToCollection(collectionsByID.get(collectionID)!, collectionFiles) + await addToCollection( + collectionsByID.get(collectionID)!, + collectionFiles, + ); tickProgress(); } // Process the removes. if (filesToTrash.length) { - // await trashFiles(filesToTrash); - - // onProgress( - // ((selectedDuplicateGroups.length - i++) / - // selectedDuplicateGroups.length) * - // -100, - // ); + await moveToTrash(filesToTrash); + tickProgress(); } + return new Set(selectedDuplicateGroups.map((g) => g.id)); }; -/** - * Retain only file from amongst these duplicates whilst keeping the existing - * collection entries intact. - * - */ -const removeDuplicateGroup = async (duplicateGroup: DuplicateGroup) => { - console.log({ fileToRetain }); - - // const collections; - // TODO: Remove me after testing the UI - await wait(1000); -}; - /** * Find the most eligible item from amongst the duplicates to retain. * From daf3fd2a75504756f807aa3143c74c749cdfefb6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:42:41 +0530 Subject: [PATCH 09/20] Sketch --- web/apps/photos/src/pages/gallery.tsx | 4 +-- web/apps/photos/src/services/sync.ts | 41 ++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 4f03254033..2d30b1a56b 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -116,7 +116,7 @@ import { getAllLatestCollections, } from "services/collectionService"; import { syncFiles } from "services/fileService"; -import { preFileInfoSync, sync } from "services/sync"; +import { preCollectionsAndFilesSync, sync } from "services/sync"; import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; import { isTokenValid } from "services/userService"; @@ -566,7 +566,7 @@ export default function Gallery() { throw new Error(CustomError.SESSION_EXPIRED); } !silent && showLoadingBar(); - await preFileInfoSync(); + await preCollectionsAndFilesSync(); const allCollections = await getAllLatestCollections(); const [hiddenCollections, collections] = splitByPredicate( allCollections, diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 065fa45917..7101c3ca2d 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -1,11 +1,16 @@ +import { isHiddenCollection } from "@/new/photos/services/collection"; import { isMLSupported, mlStatusSync, mlSync } from "@/new/photos/services/ml"; import { searchDataSync } from "@/new/photos/services/search"; import { syncSettings } from "@/new/photos/services/settings"; +import { splitByPredicate } from "@/utils/array"; +import { getAllLatestCollections } from "./collectionService"; +import { syncFiles } from "./fileService"; +import { syncTrash } from "./trashService"; /** * Part 1 of {@link sync}. See TODO below for why this is split. */ -export const preFileInfoSync = async () => { +export const preCollectionsAndFilesSync = async () => { await Promise.all([syncSettings(), isMLSupported && mlStatusSync()]); }; @@ -34,3 +39,37 @@ export const sync = async () => { // for it to finish. void mlSync(); }; + +/** + * Sync our local file and collection state with remote. + * + * This is a subset of {@link sync}, independently exposed for use at times when + * we only want to sync collections and files (e.g. we just made some API + * request that modified collections or files, and so now want to sync our local + * changes to match remote). + * + * A bespoke version of this in currently used by the gallery component when it + * syncs - it needs a broken down, bespoke version because it also keeps local + * state variables that need to be updated with the various callbacks that we + * ignore in this version. + */ +export const syncFilesAndCollections = async () => { + const allCollections = await getAllLatestCollections(); + const [hiddenCollections, normalCollections] = splitByPredicate( + allCollections, + isHiddenCollection, + ); + await syncFiles( + "normal", + normalCollections, + () => {}, + () => {}, + ); + await syncFiles( + "hidden", + hiddenCollections, + () => {}, + () => {}, + ); + await syncTrash(allCollections, () => {}); +}; From dca6e02286a85b2fe31a897f0090d30357af7c23 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:52:18 +0530 Subject: [PATCH 10/20] Move --- web/apps/photos/src/services/fileService.ts | 4 +- .../src/services/publicCollectionService.ts | 3 +- web/apps/photos/src/services/trashService.ts | 8 ++- .../src/services/upload/uploadManager.ts | 8 ++- web/apps/photos/src/utils/file/index.ts | 63 ----------------- web/packages/media/file.ts | 68 +++++++++++++++++++ 6 files changed, 83 insertions(+), 71 deletions(-) diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index ffde9fd9a1..2300cb0a98 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -2,8 +2,9 @@ import { encryptMetadataJSON } from "@/base/crypto"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import type { Collection } from "@/media/collection"; -import type { EncryptedMagicMetadata } from "@/media/file"; import { + type EncryptedMagicMetadata, + decryptFile, EncryptedEnteFile, EnteFile, FileWithUpdatedMagicMetadata, @@ -18,7 +19,6 @@ import { import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import exportService from "services/export"; -import { decryptFile } from "utils/file"; import { getCollectionLastSyncTime, setCollectionLastSyncTime, diff --git a/web/apps/photos/src/services/publicCollectionService.ts b/web/apps/photos/src/services/publicCollectionService.ts index bca52c733a..ad0d24bac1 100644 --- a/web/apps/photos/src/services/publicCollectionService.ts +++ b/web/apps/photos/src/services/publicCollectionService.ts @@ -6,12 +6,11 @@ import type { CollectionPublicMagicMetadata, } from "@/media/collection"; import type { EncryptedEnteFile, EnteFile } from "@/media/file"; -import { mergeMetadata } from "@/media/file"; +import { decryptFile, mergeMetadata } from "@/media/file"; import { sortFiles } from "@/new/photos/services/files"; import { CustomError, parseSharingErrorCodes } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; -import { decryptFile } from "utils/file"; const PUBLIC_COLLECTION_FILES_TABLE = "public-collection-files"; const PUBLIC_COLLECTIONS_TABLE = "public-collections"; diff --git a/web/apps/photos/src/services/trashService.ts b/web/apps/photos/src/services/trashService.ts index eb9aa67f0f..62e93d5664 100644 --- a/web/apps/photos/src/services/trashService.ts +++ b/web/apps/photos/src/services/trashService.ts @@ -1,7 +1,12 @@ import log from "@/base/log"; import { apiURL } from "@/base/origins"; import type { Collection } from "@/media/collection"; -import { EncryptedTrashItem, Trash, type EnteFile } from "@/media/file"; +import { + decryptFile, + EncryptedTrashItem, + Trash, + type EnteFile, +} from "@/media/file"; import { getLocalTrash, getTrashedFiles, @@ -10,7 +15,6 @@ import { import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { decryptFile } from "utils/file"; import { getCollection } from "./collectionService"; const TRASH_TIME = "trash-time"; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 474a4645f2..3f0cd395a9 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -8,7 +8,11 @@ import type { Electron } from "@/base/types/ipc"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; import { shouldDisableCFUploadProxy } from "@/gallery/services/upload"; import type { Collection } from "@/media/collection"; -import { EncryptedEnteFile, EnteFile } from "@/media/file"; +import { + decryptFile, + type EncryptedEnteFile, + type EnteFile, +} from "@/media/file"; import type { ParsedMetadata } from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { potentialFileTypeFromExtension } from "@/media/live-photo"; @@ -28,7 +32,7 @@ import { getPublicCollectionUID, } from "services/publicCollectionService"; import watcher from "services/watch"; -import { decryptFile, getUserOwnedFiles } from "utils/file"; +import { getUserOwnedFiles } from "utils/file"; import { getMetadataJSONMapKeyForJSON, tryParseTakeoutMetadataJSON, diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 85643640c7..c229dcce45 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,4 +1,3 @@ -import { sharedCryptoWorker } from "@/base/crypto"; import { joinPath } from "@/base/file-name"; import log from "@/base/log"; import { type Electron } from "@/base/types/ipc"; @@ -7,9 +6,7 @@ import { downloadManager } from "@/gallery/services/download"; import { detectFileTypeInfo } from "@/gallery/utils/detect-type"; import { writeStream } from "@/gallery/utils/native-stream"; import { - EncryptedEnteFile, EnteFile, - FileMagicMetadata, FileMagicMetadataProps, FilePublicMagicMetadata, FilePublicMagicMetadataProps, @@ -103,66 +100,6 @@ export function getSelectedFiles( return files.filter((file) => selectedFilesIDs.has(file.id)); } -export async function decryptFile( - file: EncryptedEnteFile, - collectionKey: string, -): Promise { - try { - const worker = await sharedCryptoWorker(); - const { - encryptedKey, - keyDecryptionNonce, - metadata, - magicMetadata, - pubMagicMetadata, - ...restFileProps - } = file; - const fileKey = await worker.decryptB64( - encryptedKey, - keyDecryptionNonce, - collectionKey, - ); - const fileMetadata = await worker.decryptMetadataJSON({ - encryptedDataB64: metadata.encryptedData, - decryptionHeaderB64: metadata.decryptionHeader, - keyB64: fileKey, - }); - let fileMagicMetadata: FileMagicMetadata; - let filePubMagicMetadata: FilePublicMagicMetadata; - if (magicMetadata?.data) { - fileMagicMetadata = { - ...file.magicMetadata, - data: await worker.decryptMetadataJSON({ - encryptedDataB64: magicMetadata.data, - decryptionHeaderB64: magicMetadata.header, - keyB64: fileKey, - }), - }; - } - if (pubMagicMetadata?.data) { - filePubMagicMetadata = { - ...pubMagicMetadata, - data: await worker.decryptMetadataJSON({ - encryptedDataB64: pubMagicMetadata.data, - decryptionHeaderB64: pubMagicMetadata.header, - keyB64: fileKey, - }), - }; - } - return { - ...restFileProps, - key: fileKey, - // @ts-expect-error TODO: Need to use zod here. - metadata: fileMetadata, - magicMetadata: fileMagicMetadata, - pubMagicMetadata: filePubMagicMetadata, - }; - } catch (e) { - log.error("file decryption failed", e); - throw e; - } -} - export async function changeFilesVisibility( files: EnteFile[], visibility: ItemVisibility, diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts index 1082ca5280..298ccc7592 100644 --- a/web/packages/media/file.ts +++ b/web/packages/media/file.ts @@ -1,3 +1,5 @@ +import { sharedCryptoWorker } from "@/base/crypto"; +import log from "@/base/log"; import { type Metadata, ItemVisibility } from "./file-metadata"; // TODO: Audit this file. @@ -283,6 +285,72 @@ export const fileLogID = (file: EnteFile) => // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition `file ${file.metadata.title ?? "-"} (${file.id})`; +export async function decryptFile( + file: EncryptedEnteFile, + collectionKey: string, +): Promise { + try { + const worker = await sharedCryptoWorker(); + const { + encryptedKey, + keyDecryptionNonce, + metadata, + magicMetadata, + pubMagicMetadata, + ...restFileProps + } = file; + const fileKey = await worker.decryptB64( + encryptedKey, + keyDecryptionNonce, + collectionKey, + ); + const fileMetadata = await worker.decryptMetadataJSON({ + encryptedDataB64: metadata.encryptedData, + decryptionHeaderB64: metadata.decryptionHeader, + keyB64: fileKey, + }); + let fileMagicMetadata: FileMagicMetadata; + let filePubMagicMetadata: FilePublicMagicMetadata; + /* eslint-disable @typescript-eslint/no-unnecessary-condition */ + if (magicMetadata?.data) { + fileMagicMetadata = { + ...file.magicMetadata, + // @ts-expect-error TODO update types + data: await worker.decryptMetadataJSON({ + encryptedDataB64: magicMetadata.data, + decryptionHeaderB64: magicMetadata.header, + keyB64: fileKey, + }), + }; + } + /* eslint-disable @typescript-eslint/no-unnecessary-condition */ + if (pubMagicMetadata?.data) { + filePubMagicMetadata = { + ...pubMagicMetadata, + // @ts-expect-error TODO update types + data: await worker.decryptMetadataJSON({ + encryptedDataB64: pubMagicMetadata.data, + decryptionHeaderB64: pubMagicMetadata.header, + keyB64: fileKey, + }), + }; + } + return { + ...restFileProps, + key: fileKey, + // @ts-expect-error TODO: Need to use zod here. + metadata: fileMetadata, + // @ts-expect-error TODO update types + magicMetadata: fileMagicMetadata, + // @ts-expect-error TODO update types + pubMagicMetadata: filePubMagicMetadata, + }; + } catch (e) { + log.error("file decryption failed", e); + throw e; + } +} + /** * Update the immutable fields of an (in-memory) {@link EnteFile} with any edits * that the user has made to their corresponding mutable metadata fields. From 3e23ff9c9b9e05511f97cb173574ea63e420f0e1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 17:53:55 +0530 Subject: [PATCH 11/20] Move --- web/apps/photos/src/services/collectionService.ts | 12 +----------- web/apps/photos/src/services/fileService.ts | 8 ++++---- web/packages/new/photos/services/collections.ts | 11 +++++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index a26c684d30..8e36d9903a 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -31,6 +31,7 @@ import { import { getAllLocalCollections, getLocalCollections, + removeCollectionLastSyncTime, } from "@/new/photos/services/collections"; import { getLocalFiles, @@ -65,17 +66,6 @@ const FAVORITE_COLLECTION_NAME = "Favorites"; const REQUEST_BATCH_SIZE = 1000; -export const getCollectionLastSyncTime = async (collection: Collection) => - (await localForage.getItem(`${collection.id}-time`)) ?? 0; - -export const setCollectionLastSyncTime = async ( - collection: Collection, - time: number, -) => await localForage.setItem(`${collection.id}-time`, time); - -export const removeCollectionLastSyncTime = async (collection: Collection) => - await localForage.removeItem(`${collection.id}-time`); - const getCollectionWithSecrets = async ( collection: EncryptedCollection, masterKey: string, diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index 2300cb0a98..3c41d700dc 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -10,6 +10,10 @@ import { FileWithUpdatedMagicMetadata, FileWithUpdatedPublicMagicMetadata, } from "@/media/file"; +import { + getCollectionLastSyncTime, + setCollectionLastSyncTime, +} from "@/new/photos/services/collections"; import { clearCachedThumbnailsIfChanged, getLatestVersionFiles, @@ -19,10 +23,6 @@ import { import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import exportService from "services/export"; -import { - getCollectionLastSyncTime, - setCollectionLastSyncTime, -} from "./collectionService"; /** * Fetch all files of the given {@link type}, belonging to the given diff --git a/web/packages/new/photos/services/collections.ts b/web/packages/new/photos/services/collections.ts index 44f6b9f868..0d838c993c 100644 --- a/web/packages/new/photos/services/collections.ts +++ b/web/packages/new/photos/services/collections.ts @@ -18,3 +18,14 @@ export const getAllLocalCollections = async (): Promise => { (await localForage.getItem(COLLECTION_TABLE)) ?? []; return collections; }; + +export const getCollectionLastSyncTime = async (collection: Collection) => + (await localForage.getItem(`${collection.id}-time`)) ?? 0; + +export const setCollectionLastSyncTime = async ( + collection: Collection, + time: number, +) => await localForage.setItem(`${collection.id}-time`, time); + +export const removeCollectionLastSyncTime = async (collection: Collection) => + await localForage.removeItem(`${collection.id}-time`); From adeab53d3bd4154b24ba8342962719c7ebd43aa4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:03:32 +0530 Subject: [PATCH 12/20] Move --- web/apps/photos/src/pages/gallery.tsx | 9 +- web/apps/photos/src/services/fileService.ts | 121 ------------------ web/packages/new/photos/services/files.ts | 129 +++++++++++++++++++- 3 files changed, 134 insertions(+), 125 deletions(-) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 2d30b1a56b..15e229fdb9 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -40,6 +40,7 @@ import { getLocalFiles, getLocalTrashedFiles, sortFiles, + syncFiles, } from "@/new/photos/services/files"; import { filterSearchableFiles, @@ -115,7 +116,7 @@ import { createUnCategorizedCollection, getAllLatestCollections, } from "services/collectionService"; -import { syncFiles } from "services/fileService"; +import exportService from "services/export"; import { preCollectionsAndFilesSync, sync } from "services/sync"; import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; @@ -577,13 +578,13 @@ export default function Gallery() { collections, hiddenCollections, }); - await syncFiles( + const didUpdateNormalFiles = await syncFiles( "normal", collections, (files) => dispatch({ type: "setFiles", files }), (files) => dispatch({ type: "fetchFiles", files }), ); - await syncFiles( + const didUpdateHiddenFiles = await syncFiles( "hidden", hiddenCollections, (hiddenFiles) => @@ -591,6 +592,8 @@ export default function Gallery() { (hiddenFiles) => dispatch({ type: "fetchHiddenFiles", hiddenFiles }), ); + if (didUpdateNormalFiles || didUpdateHiddenFiles) + exportService.onLocalFilesUpdated(); await syncTrash(allCollections, (trashedFiles: EnteFile[]) => dispatch({ type: "setTrashedFiles", trashedFiles }), ); diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index 3c41d700dc..f948e45067 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -1,134 +1,13 @@ import { encryptMetadataJSON } from "@/base/crypto"; -import log from "@/base/log"; import { apiURL } from "@/base/origins"; -import type { Collection } from "@/media/collection"; import { type EncryptedMagicMetadata, - decryptFile, - EncryptedEnteFile, EnteFile, FileWithUpdatedMagicMetadata, FileWithUpdatedPublicMagicMetadata, } from "@/media/file"; -import { - getCollectionLastSyncTime, - setCollectionLastSyncTime, -} from "@/new/photos/services/collections"; -import { - clearCachedThumbnailsIfChanged, - getLatestVersionFiles, - getLocalFiles, - setLocalFiles, -} from "@/new/photos/services/files"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import exportService from "services/export"; - -/** - * Fetch all files of the given {@link type}, belonging to the given - * {@link collections}, from remote and update our local database. - * - * If this is the initial read, or if the count of files we have differs from - * the state of the local database (these two are expected to be the same case), - * then the {@link onResetFiles} callback is invoked to give the caller a chance - * to bring its state up to speed. - * - * In addition to updating the local database, it also calls the provided - * {@link onFetchFiles} callback with the latest decrypted files after each - * batch the new and/or updated files are received from remote. - */ -export const syncFiles = async ( - type: "normal" | "hidden", - collections: Collection[], - onResetFiles: (fs: EnteFile[]) => void, - onFetchFiles: (fs: EnteFile[]) => void, -) => { - const localFiles = await getLocalFiles(type); - let files = await removeDeletedCollectionFiles(collections, localFiles); - let didUpdateFiles = false; - if (files.length !== localFiles.length) { - await setLocalFiles(type, files); - onResetFiles(files); - didUpdateFiles = true; - } - for (const collection of collections) { - if (!getToken()) { - continue; - } - const lastSyncTime = await getCollectionLastSyncTime(collection); - if (collection.updationTime === lastSyncTime) { - continue; - } - - const newFiles = await getFiles(collection, lastSyncTime, onFetchFiles); - await clearCachedThumbnailsIfChanged(localFiles, newFiles); - files = getLatestVersionFiles([...files, ...newFiles]); - await setLocalFiles(type, files); - didUpdateFiles = true; - setCollectionLastSyncTime(collection, collection.updationTime); - } - if (didUpdateFiles) exportService.onLocalFilesUpdated(); -}; - -export const getFiles = async ( - collection: Collection, - sinceTime: number, - onFetchFiles: (fs: EnteFile[]) => void, -): Promise => { - try { - let decryptedFiles: EnteFile[] = []; - let time = sinceTime; - let resp; - do { - const token = getToken(); - if (!token) { - break; - } - resp = await HTTPService.get( - await apiURL("/collections/v2/diff"), - { - collectionID: collection.id, - sinceTime: time, - }, - { - "X-Auth-Token": token, - }, - ); - - const newDecryptedFilesBatch = await Promise.all( - resp.data.diff.map(async (file: EncryptedEnteFile) => { - if (!file.isDeleted) { - return await decryptFile(file, collection.key); - } else { - return file; - } - }) as Promise[], - ); - decryptedFiles = [...decryptedFiles, ...newDecryptedFilesBatch]; - - onFetchFiles(decryptedFiles); - if (resp.data.diff.length) { - time = resp.data.diff.slice(-1)[0].updationTime; - } - } while (resp.data.hasMore); - return decryptedFiles; - } catch (e) { - log.error("Get files failed", e); - throw e; - } -}; - -const removeDeletedCollectionFiles = async ( - collections: Collection[], - files: EnteFile[], -) => { - const syncedCollectionIds = new Set(); - for (const collection of collections) { - syncedCollectionIds.add(collection.id); - } - files = files.filter((file) => syncedCollectionIds.has(file.collectionID)); - return files; -}; export interface UpdateMagicMetadataRequest { id: number; diff --git a/web/packages/new/photos/services/files.ts b/web/packages/new/photos/services/files.ts index 6c3c61cf7e..bca53f6132 100644 --- a/web/packages/new/photos/services/files.ts +++ b/web/packages/new/photos/services/files.ts @@ -1,7 +1,22 @@ import { blobCache } from "@/base/blob-cache"; -import { mergeMetadata, type EnteFile, type Trash } from "@/media/file"; +import log from "@/base/log"; +import { apiURL } from "@/base/origins"; +import type { Collection } from "@/media/collection"; +import { + decryptFile, + mergeMetadata, + type EncryptedEnteFile, + type EnteFile, + type Trash, +} from "@/media/file"; import { metadataHash } from "@/media/file-metadata"; +import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; +import { getToken } from "@ente/shared/storage/localStorage/helpers"; +import { + getCollectionLastSyncTime, + setCollectionLastSyncTime, +} from "./collections"; const FILES_TABLE = "files"; const HIDDEN_FILES_TABLE = "hidden-files"; @@ -37,6 +52,118 @@ export const setLocalFiles = async ( await localForage.setItem(tableName, files); }; +/** + * Fetch all files of the given {@link type}, belonging to the given + * {@link collections}, from remote and update our local database. + * + * If this is the initial read, or if the count of files we have differs from + * the state of the local database (these two are expected to be the same case), + * then the {@link onResetFiles} callback is invoked to give the caller a chance + * to bring its state up to speed. + * + * In addition to updating the local database, it also calls the provided + * {@link onFetchFiles} callback with the latest decrypted files after each + * batch the new and/or updated files are received from remote. + * + * @returns true if one or more files were updated locally, false otherwise. + */ +export const syncFiles = async ( + type: "normal" | "hidden", + collections: Collection[], + onResetFiles: (fs: EnteFile[]) => void, + onFetchFiles: (fs: EnteFile[]) => void, +) => { + const localFiles = await getLocalFiles(type); + let files = removeDeletedCollectionFiles(collections, localFiles); + let didUpdateFiles = false; + if (files.length !== localFiles.length) { + await setLocalFiles(type, files); + onResetFiles(files); + didUpdateFiles = true; + } + for (const collection of collections) { + if (!getToken()) { + continue; + } + const lastSyncTime = await getCollectionLastSyncTime(collection); + if (collection.updationTime === lastSyncTime) { + continue; + } + + const newFiles = await getFiles(collection, lastSyncTime, onFetchFiles); + await clearCachedThumbnailsIfChanged(localFiles, newFiles); + files = getLatestVersionFiles([...files, ...newFiles]); + await setLocalFiles(type, files); + didUpdateFiles = true; + await setCollectionLastSyncTime(collection, collection.updationTime); + } + return didUpdateFiles; +}; + +export const getFiles = async ( + collection: Collection, + sinceTime: number, + onFetchFiles: (fs: EnteFile[]) => void, +): Promise => { + try { + let decryptedFiles: EnteFile[] = []; + let time = sinceTime; + let resp; + do { + const token = getToken(); + if (!token) { + break; + } + resp = await HTTPService.get( + await apiURL("/collections/v2/diff"), + { + collectionID: collection.id, + sinceTime: time, + }, + { + "X-Auth-Token": token, + }, + ); + + const newDecryptedFilesBatch = await Promise.all( + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + resp.data.diff.map(async (file: EncryptedEnteFile) => { + if (!file.isDeleted) { + return await decryptFile(file, collection.key); + } else { + return file; + } + }) as Promise[], + ); + decryptedFiles = [...decryptedFiles, ...newDecryptedFilesBatch]; + + onFetchFiles(decryptedFiles); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if (resp.data.diff.length) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access + time = resp.data.diff.slice(-1)[0].updationTime; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + } while (resp.data.hasMore); + return decryptedFiles; + } catch (e) { + log.error("Get files failed", e); + throw e; + } +}; + +const removeDeletedCollectionFiles = ( + collections: Collection[], + files: EnteFile[], +) => { + const syncedCollectionIds = new Set(); + for (const collection of collections) { + syncedCollectionIds.add(collection.id); + } + files = files.filter((file) => syncedCollectionIds.has(file.collectionID)); + return files; +}; + /** * Sort the given list of {@link EnteFile}s in place. * From 746c85bc9ffa2b74d9a23992f795c475fd190d4e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:20:49 +0530 Subject: [PATCH 13/20] Move --- .../photos/src/components/Upload/Uploader.tsx | 2 +- web/apps/photos/src/pages/deduplicate.tsx | 8 +- web/apps/photos/src/pages/gallery.tsx | 6 +- .../photos/src/services/collectionService.ts | 246 +--------------- web/apps/photos/src/services/sync.ts | 4 +- web/apps/photos/src/services/trashService.ts | 2 +- .../new/photos/services/collections.ts | 269 +++++++++++++++++- 7 files changed, 282 insertions(+), 255 deletions(-) diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index 1c3c17e108..f0461e76b6 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -9,6 +9,7 @@ import { UploaderNameInput } from "@/new/albums/components/UploaderNameInput"; import { CollectionMappingChoice } from "@/new/photos/components/CollectionMappingChoice"; import type { CollectionSelectorAttributes } from "@/new/photos/components/CollectionSelector"; import { downloadAppDialogAttributes } from "@/new/photos/components/utils/download"; +import { getLatestCollections } from "@/new/photos/services/collections"; import { exportMetadataDirectoryName } from "@/new/photos/services/export"; import type { FileAndPath, @@ -26,7 +27,6 @@ import { t } from "i18next"; import { GalleryContext } from "pages/gallery"; import { useContext, useEffect, useRef, useState } from "react"; import { Trans } from "react-i18next"; -import { getLatestCollections } from "services/collectionService"; import { getPublicCollectionUID, getPublicCollectionUploaderName, diff --git a/web/apps/photos/src/pages/deduplicate.tsx b/web/apps/photos/src/pages/deduplicate.tsx index 4b1d16630a..6defce11f2 100644 --- a/web/apps/photos/src/pages/deduplicate.tsx +++ b/web/apps/photos/src/pages/deduplicate.tsx @@ -3,10 +3,14 @@ import { ActivityIndicator } from "@/base/components/mui/ActivityIndicator"; import { errorDialogAttributes } from "@/base/components/utils/dialog"; import log from "@/base/log"; import { ALL_SECTION, moveToTrash } from "@/new/photos/services/collection"; -import { getLocalCollections } from "@/new/photos/services/collections"; +import { + getAllLatestCollections, + getLocalCollections, +} from "@/new/photos/services/collections"; import { createFileCollectionIDs, getLocalFiles, + syncFiles, } from "@/new/photos/services/files"; import { useAppContext } from "@/new/photos/types/context"; import { VerticallyCentered } from "@ente/shared/components/Container"; @@ -22,9 +26,7 @@ import PhotoFrame from "components/PhotoFrame"; import { t } from "i18next"; import { default as Router, default as router } from "next/router"; import { createContext, useEffect, useState } from "react"; -import { getAllLatestCollections } from "services/collectionService"; import { Duplicate, getDuplicates } from "services/deduplicationService"; -import { syncFiles } from "services/fileService"; import { syncTrash } from "services/trashService"; import { SelectedState } from "types/gallery"; import { getSelectedFiles } from "utils/file"; diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 15e229fdb9..1b6f41b299 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -35,7 +35,10 @@ import { isHiddenCollection, } from "@/new/photos/services/collection"; import { areOnlySystemCollections } from "@/new/photos/services/collection/ui"; -import { getAllLocalCollections } from "@/new/photos/services/collections"; +import { + getAllLatestCollections, + getAllLocalCollections, +} from "@/new/photos/services/collections"; import { getLocalFiles, getLocalTrashedFiles, @@ -114,7 +117,6 @@ import { constructUserIDToEmailMap, createAlbum, createUnCategorizedCollection, - getAllLatestCollections, } from "services/collectionService"; import exportService from "services/export"; import { preCollectionsAndFilesSync, sync } from "services/sync"; diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index 8e36d9903a..a8dc225337 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -6,7 +6,6 @@ import { CollectionMagicMetadata, CollectionMagicMetadataProps, CollectionPublicMagicMetadata, - CollectionShareeMagicMetadata, CollectionType, CreatePublicAccessTokenRequest, EncryptedCollection, @@ -20,7 +19,6 @@ import { ItemVisibility } from "@/media/file-metadata"; import { addToCollection, isDefaultHiddenCollection, - isHiddenCollection, moveToCollection, } from "@/new/photos/services/collection"; import type { CollectionSummary } from "@/new/photos/services/collection/ui"; @@ -29,9 +27,8 @@ import { CollectionsSortBy, } from "@/new/photos/services/collection/ui"; import { - getAllLocalCollections, + getCollectionWithSecrets, getLocalCollections, - removeCollectionLastSyncTime, } from "@/new/photos/services/collections"; import { getLocalFiles, @@ -42,7 +39,6 @@ import { updateMagicMetadata } from "@/new/photos/services/magic-metadata"; import type { FamilyData } from "@/new/photos/services/user-details"; import { batch } from "@/utils/array"; import HTTPService from "@ente/shared/network/HTTPService"; -import localForage from "@ente/shared/storage/localForage"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import { getActualKey } from "@ente/shared/user"; @@ -55,252 +51,12 @@ import { import { UpdateMagicMetadataRequest } from "./fileService"; import { getPublicKey } from "./userService"; -const COLLECTION_TABLE = "collections"; - -const COLLECTION_UPDATION_TIME = "collection-updation-time"; -const HIDDEN_COLLECTION_IDS = "hidden-collection-ids"; - const UNCATEGORIZED_COLLECTION_NAME = "Uncategorized"; export const HIDDEN_COLLECTION_NAME = ".hidden"; const FAVORITE_COLLECTION_NAME = "Favorites"; const REQUEST_BATCH_SIZE = 1000; -const getCollectionWithSecrets = async ( - collection: EncryptedCollection, - masterKey: string, -): Promise => { - const cryptoWorker = await sharedCryptoWorker(); - const userID = getData(LS_KEYS.USER).id; - let collectionKey: string; - if (collection.owner.id === userID) { - collectionKey = await cryptoWorker.decryptB64( - collection.encryptedKey, - collection.keyDecryptionNonce, - masterKey, - ); - } else { - const keyAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); - const secretKey = await cryptoWorker.decryptB64( - keyAttributes.encryptedSecretKey, - keyAttributes.secretKeyDecryptionNonce, - masterKey, - ); - collectionKey = await cryptoWorker.boxSealOpen( - collection.encryptedKey, - keyAttributes.publicKey, - secretKey, - ); - } - const collectionName = - collection.name || - (await cryptoWorker.decryptToUTF8( - collection.encryptedName, - collection.nameDecryptionNonce, - collectionKey, - )); - - let collectionMagicMetadata: CollectionMagicMetadata; - if (collection.magicMetadata?.data) { - collectionMagicMetadata = { - ...collection.magicMetadata, - data: await cryptoWorker.decryptMetadataJSON({ - encryptedDataB64: collection.magicMetadata.data, - decryptionHeaderB64: collection.magicMetadata.header, - keyB64: collectionKey, - }), - }; - } - let collectionPublicMagicMetadata: CollectionPublicMagicMetadata; - if (collection.pubMagicMetadata?.data) { - collectionPublicMagicMetadata = { - ...collection.pubMagicMetadata, - data: await cryptoWorker.decryptMetadataJSON({ - encryptedDataB64: collection.pubMagicMetadata.data, - decryptionHeaderB64: collection.pubMagicMetadata.header, - keyB64: collectionKey, - }), - }; - } - - let collectionShareeMagicMetadata: CollectionShareeMagicMetadata; - if (collection.sharedMagicMetadata?.data) { - collectionShareeMagicMetadata = { - ...collection.sharedMagicMetadata, - data: await cryptoWorker.decryptMetadataJSON({ - encryptedDataB64: collection.sharedMagicMetadata.data, - decryptionHeaderB64: collection.sharedMagicMetadata.header, - keyB64: collectionKey, - }), - }; - } - - return { - ...collection, - name: collectionName, - key: collectionKey, - magicMetadata: collectionMagicMetadata, - pubMagicMetadata: collectionPublicMagicMetadata, - sharedMagicMetadata: collectionShareeMagicMetadata, - }; -}; - -const getCollections = async ( - token: string, - sinceTime: number, - key: string, -): Promise => { - try { - const resp = await HTTPService.get( - await apiURL("/collections/v2"), - { - sinceTime, - }, - { "X-Auth-Token": token }, - ); - const decryptedCollections: Collection[] = await Promise.all( - resp.data.collections.map( - async (collection: EncryptedCollection) => { - if (collection.isDeleted) { - return collection; - } - try { - return await getCollectionWithSecrets(collection, key); - } catch (e) { - log.error( - `decryption failed for collection with ID ${collection.id}`, - e, - ); - return collection; - } - }, - ), - ); - // only allow deleted or collection with key, filtering out collection whose decryption failed - const collections = decryptedCollections.filter( - (collection) => collection.isDeleted || collection.key, - ); - return collections; - } catch (e) { - log.error("getCollections failed", e); - throw e; - } -}; - -export const getCollectionUpdationTime = async (): Promise => - (await localForage.getItem(COLLECTION_UPDATION_TIME)) ?? 0; - -export const getHiddenCollectionIDs = async (): Promise => - (await localForage.getItem(HIDDEN_COLLECTION_IDS)) ?? []; - -export const getLatestCollections = async ( - type: "normal" | "hidden" = "normal", -): Promise => { - const collections = await getAllLatestCollections(); - return type == "normal" - ? collections.filter((c) => !isHiddenCollection(c)) - : collections.filter((c) => isHiddenCollection(c)); -}; - -export const getAllLatestCollections = async (): Promise => { - const collections = await syncCollections(); - return collections; -}; - -export const syncCollections = async () => { - const localCollections = await getAllLocalCollections(); - let lastCollectionUpdationTime = await getCollectionUpdationTime(); - const hiddenCollectionIDs = await getHiddenCollectionIDs(); - const token = getToken(); - const key = await getActualKey(); - const updatedCollections = - (await getCollections(token, lastCollectionUpdationTime, key)) ?? []; - if (updatedCollections.length === 0) { - return localCollections; - } - const allCollectionsInstances = [ - ...localCollections, - ...updatedCollections, - ]; - const latestCollectionsInstances = new Map(); - allCollectionsInstances.forEach((collection) => { - if ( - !latestCollectionsInstances.has(collection.id) || - latestCollectionsInstances.get(collection.id).updationTime < - collection.updationTime - ) { - latestCollectionsInstances.set(collection.id, collection); - } - }); - - const collections: Collection[] = []; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for (const [_, collection] of latestCollectionsInstances) { - const isDeletedCollection = collection.isDeleted; - const isNewlyHiddenCollection = - isHiddenCollection(collection) && - !hiddenCollectionIDs.includes(collection.id); - const isNewlyUnHiddenCollection = - !isHiddenCollection(collection) && - hiddenCollectionIDs.includes(collection.id); - if ( - isDeletedCollection || - isNewlyHiddenCollection || - isNewlyUnHiddenCollection - ) { - removeCollectionLastSyncTime(collection); - } - if (isDeletedCollection) { - continue; - } - collections.push(collection); - lastCollectionUpdationTime = Math.max( - lastCollectionUpdationTime, - collection.updationTime, - ); - } - - const updatedHiddenCollectionIDs = collections - .filter((collection) => isHiddenCollection(collection)) - .map((collection) => collection.id); - - await localForage.setItem(COLLECTION_TABLE, collections); - await localForage.setItem( - COLLECTION_UPDATION_TIME, - lastCollectionUpdationTime, - ); - await localForage.setItem( - HIDDEN_COLLECTION_IDS, - updatedHiddenCollectionIDs, - ); - return collections; -}; - -export const getCollection = async ( - collectionID: number, -): Promise => { - try { - const token = getToken(); - if (!token) { - return; - } - const resp = await HTTPService.get( - await apiURL(`/collections/${collectionID}`), - null, - { "X-Auth-Token": token }, - ); - const key = await getActualKey(); - const collectionWithSecrets = await getCollectionWithSecrets( - resp.data?.collection, - key, - ); - return collectionWithSecrets; - } catch (e) { - log.error("failed to get collection", e); - throw e; - } -}; - export const createAlbum = (albumName: string) => { return createCollection(albumName, CollectionType.album); }; diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 7101c3ca2d..2bba28055f 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -1,10 +1,10 @@ import { isHiddenCollection } from "@/new/photos/services/collection"; +import { getAllLatestCollections } from "@/new/photos/services/collections"; +import { syncFiles } from "@/new/photos/services/files"; import { isMLSupported, mlStatusSync, mlSync } from "@/new/photos/services/ml"; import { searchDataSync } from "@/new/photos/services/search"; import { syncSettings } from "@/new/photos/services/settings"; import { splitByPredicate } from "@/utils/array"; -import { getAllLatestCollections } from "./collectionService"; -import { syncFiles } from "./fileService"; import { syncTrash } from "./trashService"; /** diff --git a/web/apps/photos/src/services/trashService.ts b/web/apps/photos/src/services/trashService.ts index 62e93d5664..9d61cea1ed 100644 --- a/web/apps/photos/src/services/trashService.ts +++ b/web/apps/photos/src/services/trashService.ts @@ -7,6 +7,7 @@ import { Trash, type EnteFile, } from "@/media/file"; +import { getCollection } from "@/new/photos/services/collections"; import { getLocalTrash, getTrashedFiles, @@ -15,7 +16,6 @@ import { import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { getCollection } from "./collectionService"; const TRASH_TIME = "trash-time"; const DELETED_COLLECTION = "deleted-collection"; diff --git a/web/packages/new/photos/services/collections.ts b/web/packages/new/photos/services/collections.ts index 0d838c993c..3bec9b4d70 100644 --- a/web/packages/new/photos/services/collections.ts +++ b/web/packages/new/photos/services/collections.ts @@ -1,8 +1,31 @@ -import { type Collection } from "@/media/collection"; +/* eslint-disable @typescript-eslint/no-unsafe-call */ +// TODO: Audit this file +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ +/* eslint-disable @typescript-eslint/prefer-nullish-coalescing */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ + +import { sharedCryptoWorker } from "@/base/crypto"; +import log from "@/base/log"; +import { apiURL } from "@/base/origins"; +import { + type Collection, + type CollectionMagicMetadata, + type CollectionPublicMagicMetadata, + type CollectionShareeMagicMetadata, + type EncryptedCollection, +} from "@/media/collection"; +import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; +import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; +import { getToken } from "@ente/shared/storage/localStorage/helpers"; +import { getActualKey } from "@ente/shared/user"; import { isHiddenCollection } from "./collection"; const COLLECTION_TABLE = "collections"; +const HIDDEN_COLLECTION_IDS = "hidden-collection-ids"; +const COLLECTION_UPDATION_TIME = "collection-updation-time"; export const getLocalCollections = async ( type: "normal" | "hidden" = "normal", @@ -29,3 +52,247 @@ export const setCollectionLastSyncTime = async ( export const removeCollectionLastSyncTime = async (collection: Collection) => await localForage.removeItem(`${collection.id}-time`); + +export const getHiddenCollectionIDs = async (): Promise => + (await localForage.getItem(HIDDEN_COLLECTION_IDS)) ?? []; + +export const getCollectionUpdationTime = async (): Promise => + (await localForage.getItem(COLLECTION_UPDATION_TIME)) ?? 0; + +export const getLatestCollections = async ( + type: "normal" | "hidden" = "normal", +): Promise => { + const collections = await getAllLatestCollections(); + return type == "normal" + ? collections.filter((c) => !isHiddenCollection(c)) + : collections.filter((c) => isHiddenCollection(c)); +}; + +export const getAllLatestCollections = async (): Promise => { + const collections = await syncCollections(); + return collections; +}; + +export const syncCollections = async () => { + const localCollections = await getAllLocalCollections(); + let lastCollectionUpdationTime = await getCollectionUpdationTime(); + const hiddenCollectionIDs = await getHiddenCollectionIDs(); + const token = getToken(); + const key = await getActualKey(); + const updatedCollections = + (await getCollections(token, lastCollectionUpdationTime, key)) ?? []; + if (updatedCollections.length === 0) { + return localCollections; + } + const allCollectionsInstances = [ + ...localCollections, + ...updatedCollections, + ]; + const latestCollectionsInstances = new Map(); + allCollectionsInstances.forEach((collection) => { + if ( + !latestCollectionsInstances.has(collection.id) || + // @ts-expect-error TODO fixme + latestCollectionsInstances.get(collection.id).updationTime < + collection.updationTime + ) { + latestCollectionsInstances.set(collection.id, collection); + } + }); + + const collections: Collection[] = []; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, collection] of latestCollectionsInstances) { + const isDeletedCollection = collection.isDeleted; + const isNewlyHiddenCollection = + isHiddenCollection(collection) && + !hiddenCollectionIDs.includes(collection.id); + const isNewlyUnHiddenCollection = + !isHiddenCollection(collection) && + hiddenCollectionIDs.includes(collection.id); + if ( + isDeletedCollection || + isNewlyHiddenCollection || + isNewlyUnHiddenCollection + ) { + await removeCollectionLastSyncTime(collection); + } + if (isDeletedCollection) { + continue; + } + collections.push(collection); + lastCollectionUpdationTime = Math.max( + lastCollectionUpdationTime, + collection.updationTime, + ); + } + + const updatedHiddenCollectionIDs = collections + .filter((collection) => isHiddenCollection(collection)) + .map((collection) => collection.id); + + await localForage.setItem(COLLECTION_TABLE, collections); + await localForage.setItem( + COLLECTION_UPDATION_TIME, + lastCollectionUpdationTime, + ); + await localForage.setItem( + HIDDEN_COLLECTION_IDS, + updatedHiddenCollectionIDs, + ); + return collections; +}; + +const getCollections = async ( + token: string, + sinceTime: number, + key: string, +): Promise => { + try { + const resp = await HTTPService.get( + await apiURL("/collections/v2"), + { + sinceTime, + }, + { "X-Auth-Token": token }, + ); + const decryptedCollections: Collection[] = await Promise.all( + resp.data.collections.map( + async (collection: EncryptedCollection) => { + if (collection.isDeleted) { + return collection; + } + try { + return await getCollectionWithSecrets(collection, key); + } catch (e) { + log.error( + `decryption failed for collection with ID ${collection.id}`, + e, + ); + return collection; + } + }, + ), + ); + // only allow deleted or collection with key, filtering out collection whose decryption failed + const collections = decryptedCollections.filter( + (collection) => collection.isDeleted || collection.key, + ); + return collections; + } catch (e) { + log.error("getCollections failed", e); + throw e; + } +}; + +export const getCollectionWithSecrets = async ( + collection: EncryptedCollection, + masterKey: string, +): Promise => { + const cryptoWorker = await sharedCryptoWorker(); + const userID = getData(LS_KEYS.USER).id; + let collectionKey: string; + if (collection.owner.id === userID) { + collectionKey = await cryptoWorker.decryptB64( + collection.encryptedKey, + collection.keyDecryptionNonce, + masterKey, + ); + } else { + const keyAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); + const secretKey = await cryptoWorker.decryptB64( + keyAttributes.encryptedSecretKey, + keyAttributes.secretKeyDecryptionNonce, + masterKey, + ); + collectionKey = await cryptoWorker.boxSealOpen( + collection.encryptedKey, + keyAttributes.publicKey, + secretKey, + ); + } + const collectionName = + collection.name || + (await cryptoWorker.decryptToUTF8( + collection.encryptedName, + collection.nameDecryptionNonce, + collectionKey, + )); + + let collectionMagicMetadata: CollectionMagicMetadata; + if (collection.magicMetadata?.data) { + collectionMagicMetadata = { + ...collection.magicMetadata, + // @ts-expect-error TODO fixme + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.magicMetadata.data, + decryptionHeaderB64: collection.magicMetadata.header, + keyB64: collectionKey, + }), + }; + } + let collectionPublicMagicMetadata: CollectionPublicMagicMetadata; + if (collection.pubMagicMetadata?.data) { + collectionPublicMagicMetadata = { + ...collection.pubMagicMetadata, + // @ts-expect-error TODO fixme + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.pubMagicMetadata.data, + decryptionHeaderB64: collection.pubMagicMetadata.header, + keyB64: collectionKey, + }), + }; + } + + let collectionShareeMagicMetadata: CollectionShareeMagicMetadata; + if (collection.sharedMagicMetadata?.data) { + collectionShareeMagicMetadata = { + ...collection.sharedMagicMetadata, + // @ts-expect-error TODO fixme + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.sharedMagicMetadata.data, + decryptionHeaderB64: collection.sharedMagicMetadata.header, + keyB64: collectionKey, + }), + }; + } + + return { + ...collection, + name: collectionName, + key: collectionKey, + // @ts-expect-error TODO fixme + magicMetadata: collectionMagicMetadata, + // @ts-expect-error TODO fixme + pubMagicMetadata: collectionPublicMagicMetadata, + // @ts-expect-error TODO fixme + sharedMagicMetadata: collectionShareeMagicMetadata, + }; +}; + +export const getCollection = async ( + collectionID: number, +): Promise => { + try { + const token = getToken(); + if (!token) { + // @ts-expect-error TODO fixme + return; + } + const resp = await HTTPService.get( + await apiURL(`/collections/${collectionID}`), + // @ts-expect-error TODO fixme + null, + { "X-Auth-Token": token }, + ); + const key = await getActualKey(); + const collectionWithSecrets = await getCollectionWithSecrets( + resp.data?.collection, + key, + ); + return collectionWithSecrets; + } catch (e) { + log.error("failed to get collection", e); + throw e; + } +}; From aae2632b19685e790a038ade297381f87a986cab Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:28:54 +0530 Subject: [PATCH 14/20] See: [Note: strict mode migration] --- web/packages/media/file.ts | 12 ++++++--- .../new/photos/services/collections.ts | 27 ++++++++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts index 298ccc7592..13a9cd8e7b 100644 --- a/web/packages/media/file.ts +++ b/web/packages/media/file.ts @@ -315,7 +315,8 @@ export async function decryptFile( if (magicMetadata?.data) { fileMagicMetadata = { ...file.magicMetadata, - // @ts-expect-error TODO update types + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore data: await worker.decryptMetadataJSON({ encryptedDataB64: magicMetadata.data, decryptionHeaderB64: magicMetadata.header, @@ -327,7 +328,8 @@ export async function decryptFile( if (pubMagicMetadata?.data) { filePubMagicMetadata = { ...pubMagicMetadata, - // @ts-expect-error TODO update types + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore data: await worker.decryptMetadataJSON({ encryptedDataB64: pubMagicMetadata.data, decryptionHeaderB64: pubMagicMetadata.header, @@ -340,9 +342,11 @@ export async function decryptFile( key: fileKey, // @ts-expect-error TODO: Need to use zod here. metadata: fileMetadata, - // @ts-expect-error TODO update types + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore magicMetadata: fileMagicMetadata, - // @ts-expect-error TODO update types + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore pubMagicMetadata: filePubMagicMetadata, }; } catch (e) { diff --git a/web/packages/new/photos/services/collections.ts b/web/packages/new/photos/services/collections.ts index 3bec9b4d70..647f8a037c 100644 --- a/web/packages/new/photos/services/collections.ts +++ b/web/packages/new/photos/services/collections.ts @@ -92,7 +92,8 @@ export const syncCollections = async () => { allCollectionsInstances.forEach((collection) => { if ( !latestCollectionsInstances.has(collection.id) || - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore latestCollectionsInstances.get(collection.id).updationTime < collection.updationTime ) { @@ -223,7 +224,8 @@ export const getCollectionWithSecrets = async ( if (collection.magicMetadata?.data) { collectionMagicMetadata = { ...collection.magicMetadata, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore data: await cryptoWorker.decryptMetadataJSON({ encryptedDataB64: collection.magicMetadata.data, decryptionHeaderB64: collection.magicMetadata.header, @@ -235,7 +237,8 @@ export const getCollectionWithSecrets = async ( if (collection.pubMagicMetadata?.data) { collectionPublicMagicMetadata = { ...collection.pubMagicMetadata, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore data: await cryptoWorker.decryptMetadataJSON({ encryptedDataB64: collection.pubMagicMetadata.data, decryptionHeaderB64: collection.pubMagicMetadata.header, @@ -248,7 +251,8 @@ export const getCollectionWithSecrets = async ( if (collection.sharedMagicMetadata?.data) { collectionShareeMagicMetadata = { ...collection.sharedMagicMetadata, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore data: await cryptoWorker.decryptMetadataJSON({ encryptedDataB64: collection.sharedMagicMetadata.data, decryptionHeaderB64: collection.sharedMagicMetadata.header, @@ -261,11 +265,14 @@ export const getCollectionWithSecrets = async ( ...collection, name: collectionName, key: collectionKey, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore magicMetadata: collectionMagicMetadata, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore pubMagicMetadata: collectionPublicMagicMetadata, - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore sharedMagicMetadata: collectionShareeMagicMetadata, }; }; @@ -276,12 +283,14 @@ export const getCollection = async ( try { const token = getToken(); if (!token) { - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore return; } const resp = await HTTPService.get( await apiURL(`/collections/${collectionID}`), - // @ts-expect-error TODO fixme + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore null, { "X-Auth-Token": token }, ); From a292f01187f9652ecd353942c91c0cf96764932a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:42:57 +0530 Subject: [PATCH 15/20] Move --- .../Collections/CollectionHeader.tsx | 13 +- web/apps/photos/src/pages/deduplicate.tsx | 2 +- web/apps/photos/src/pages/gallery.tsx | 2 +- web/apps/photos/src/services/sync.ts | 6 +- web/apps/photos/src/services/trashService.ts | 160 ------------------ .../new/photos/services/collections.ts | 157 ++++++++++++++++- 6 files changed, 170 insertions(+), 170 deletions(-) diff --git a/web/apps/photos/src/components/Collections/CollectionHeader.tsx b/web/apps/photos/src/components/Collections/CollectionHeader.tsx index 25d0bd0261..0e48f2da78 100644 --- a/web/apps/photos/src/components/Collections/CollectionHeader.tsx +++ b/web/apps/photos/src/components/Collections/CollectionHeader.tsx @@ -50,7 +50,10 @@ import { GalleryContext } from "pages/gallery"; import React, { useCallback, useContext, useRef } from "react"; import { Trans } from "react-i18next"; import * as CollectionAPI from "services/collectionService"; -import * as TrashService from "services/trashService"; +import { + emptyTrash, + clearLocalTrash, +} from "@/new/photos/services/collections"; import { SetFilesDownloadProgressAttributesCreator } from "types/gallery"; import { changeCollectionOrder, @@ -229,13 +232,13 @@ const CollectionOptions: React.FC = ({ continue: { text: t("empty_trash"), color: "critical", - action: emptyTrash, + action: doEmptyTrash, }, }); - const emptyTrash = wrap(async () => { - await TrashService.emptyTrash(); - await TrashService.clearLocalTrash(); + const doEmptyTrash = wrap(async () => { + await emptyTrash(); + await clearLocalTrash(); setActiveCollectionID(ALL_SECTION); }); diff --git a/web/apps/photos/src/pages/deduplicate.tsx b/web/apps/photos/src/pages/deduplicate.tsx index 6defce11f2..fbeb5bcbd1 100644 --- a/web/apps/photos/src/pages/deduplicate.tsx +++ b/web/apps/photos/src/pages/deduplicate.tsx @@ -6,6 +6,7 @@ import { ALL_SECTION, moveToTrash } from "@/new/photos/services/collection"; import { getAllLatestCollections, getLocalCollections, + syncTrash, } from "@/new/photos/services/collections"; import { createFileCollectionIDs, @@ -27,7 +28,6 @@ import { t } from "i18next"; import { default as Router, default as router } from "next/router"; import { createContext, useEffect, useState } from "react"; import { Duplicate, getDuplicates } from "services/deduplicationService"; -import { syncTrash } from "services/trashService"; import { SelectedState } from "types/gallery"; import { getSelectedFiles } from "utils/file"; diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 1b6f41b299..60ae8c915d 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -38,6 +38,7 @@ import { areOnlySystemCollections } from "@/new/photos/services/collection/ui"; import { getAllLatestCollections, getAllLocalCollections, + syncTrash, } from "@/new/photos/services/collections"; import { getLocalFiles, @@ -120,7 +121,6 @@ import { } from "services/collectionService"; import exportService from "services/export"; import { preCollectionsAndFilesSync, sync } from "services/sync"; -import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; import { isTokenValid } from "services/userService"; import { diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 2bba28055f..63bb1841fc 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -1,11 +1,13 @@ import { isHiddenCollection } from "@/new/photos/services/collection"; -import { getAllLatestCollections } from "@/new/photos/services/collections"; +import { + getAllLatestCollections, + syncTrash, +} from "@/new/photos/services/collections"; import { syncFiles } from "@/new/photos/services/files"; import { isMLSupported, mlStatusSync, mlSync } from "@/new/photos/services/ml"; import { searchDataSync } from "@/new/photos/services/search"; import { syncSettings } from "@/new/photos/services/settings"; import { splitByPredicate } from "@/utils/array"; -import { syncTrash } from "./trashService"; /** * Part 1 of {@link sync}. See TODO below for why this is split. diff --git a/web/apps/photos/src/services/trashService.ts b/web/apps/photos/src/services/trashService.ts index 9d61cea1ed..e69de29bb2 100644 --- a/web/apps/photos/src/services/trashService.ts +++ b/web/apps/photos/src/services/trashService.ts @@ -1,160 +0,0 @@ -import log from "@/base/log"; -import { apiURL } from "@/base/origins"; -import type { Collection } from "@/media/collection"; -import { - decryptFile, - EncryptedTrashItem, - Trash, - type EnteFile, -} from "@/media/file"; -import { getCollection } from "@/new/photos/services/collections"; -import { - getLocalTrash, - getTrashedFiles, - TRASH, -} from "@/new/photos/services/files"; -import HTTPService from "@ente/shared/network/HTTPService"; -import localForage from "@ente/shared/storage/localForage"; -import { getToken } from "@ente/shared/storage/localStorage/helpers"; - -const TRASH_TIME = "trash-time"; -const DELETED_COLLECTION = "deleted-collection"; - -export async function getLocalDeletedCollections() { - const trashedCollections: Collection[] = - (await localForage.getItem(DELETED_COLLECTION)) || []; - const nonUndefinedCollections = trashedCollections.filter( - (collection) => !!collection, - ); - if (nonUndefinedCollections.length !== trashedCollections.length) { - await localForage.setItem(DELETED_COLLECTION, nonUndefinedCollections); - } - return nonUndefinedCollections; -} - -export async function cleanTrashCollections(fileTrash: Trash) { - const trashedCollections = await getLocalDeletedCollections(); - const neededTrashCollections = new Set( - fileTrash.map((item) => item.file.collectionID), - ); - const filterCollections = trashedCollections.filter((item) => - neededTrashCollections.has(item.id), - ); - await localForage.setItem(DELETED_COLLECTION, filterCollections); -} - -async function getLastSyncTime() { - return (await localForage.getItem(TRASH_TIME)) ?? 0; -} -export async function syncTrash( - collections: Collection[], - setTrashedFiles: (fs: EnteFile[]) => void, -): Promise { - const trash = await getLocalTrash(); - collections = [...collections, ...(await getLocalDeletedCollections())]; - const collectionMap = new Map( - collections.map((collection) => [collection.id, collection]), - ); - if (!getToken()) { - return; - } - const lastSyncTime = await getLastSyncTime(); - - const updatedTrash = await updateTrash( - collectionMap, - lastSyncTime, - setTrashedFiles, - trash, - ); - cleanTrashCollections(updatedTrash); -} - -export const updateTrash = async ( - collections: Map, - sinceTime: number, - setTrashedFiles: (fs: EnteFile[]) => void, - currentTrash: Trash, -): Promise => { - try { - let updatedTrash: Trash = [...currentTrash]; - let time = sinceTime; - - let resp; - do { - const token = getToken(); - if (!token) { - break; - } - resp = await HTTPService.get( - await apiURL("/trash/v2/diff"), - { - sinceTime: time, - }, - { - "X-Auth-Token": token, - }, - ); - // #Perf: This can be optimized by running the decryption in parallel - for (const trashItem of resp.data.diff as EncryptedTrashItem[]) { - const collectionID = trashItem.file.collectionID; - let collection = collections.get(collectionID); - if (!collection) { - collection = await getCollection(collectionID); - collections.set(collectionID, collection); - localForage.setItem(DELETED_COLLECTION, [ - ...collections.values(), - ]); - } - if (!trashItem.isDeleted && !trashItem.isRestored) { - const decryptedFile = await decryptFile( - trashItem.file, - collection.key, - ); - updatedTrash.push({ ...trashItem, file: decryptedFile }); - } else { - updatedTrash = updatedTrash.filter( - (item) => item.file.id !== trashItem.file.id, - ); - } - } - - if (resp.data.diff.length) { - time = resp.data.diff.slice(-1)[0].updatedAt; - } - - setTrashedFiles(getTrashedFiles(updatedTrash)); - await localForage.setItem(TRASH, updatedTrash); - await localForage.setItem(TRASH_TIME, time); - } while (resp.data.hasMore); - return updatedTrash; - } catch (e) { - log.error("Get trash files failed", e); - } - return currentTrash; -}; - -export const emptyTrash = async () => { - try { - const token = getToken(); - if (!token) { - return; - } - const lastUpdatedAt = await getLastSyncTime(); - - await HTTPService.post( - await apiURL("/trash/empty"), - { lastUpdatedAt }, - null, - { - "X-Auth-Token": token, - }, - ); - } catch (e) { - log.error("empty trash failed", e); - throw e; - } -}; - -export const clearLocalTrash = async () => { - await localForage.setItem(TRASH, []); -}; diff --git a/web/packages/new/photos/services/collections.ts b/web/packages/new/photos/services/collections.ts index 647f8a037c..a86e326baa 100644 --- a/web/packages/new/photos/services/collections.ts +++ b/web/packages/new/photos/services/collections.ts @@ -16,9 +16,20 @@ import { type CollectionShareeMagicMetadata, type EncryptedCollection, } from "@/media/collection"; +import { + decryptFile, + type EncryptedTrashItem, + type EnteFile, + type Trash, +} from "@/media/file"; +import { + getLocalTrash, + getTrashedFiles, + TRASH, +} from "@/new/photos/services/files"; import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; -import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; +import { getData, LS_KEYS } from "@ente/shared/storage/localStorage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import { getActualKey } from "@ente/shared/user"; import { isHiddenCollection } from "./collection"; @@ -305,3 +316,147 @@ export const getCollection = async ( throw e; } }; + +const TRASH_TIME = "trash-time"; +const DELETED_COLLECTION = "deleted-collection"; + +export async function getLocalDeletedCollections() { + const trashedCollections: Collection[] = + (await localForage.getItem(DELETED_COLLECTION)) || []; + const nonUndefinedCollections = trashedCollections.filter( + (collection) => !!collection, + ); + if (nonUndefinedCollections.length !== trashedCollections.length) { + await localForage.setItem(DELETED_COLLECTION, nonUndefinedCollections); + } + return nonUndefinedCollections; +} + +export async function cleanTrashCollections(fileTrash: Trash) { + const trashedCollections = await getLocalDeletedCollections(); + const neededTrashCollections = new Set( + fileTrash.map((item) => item.file.collectionID), + ); + const filterCollections = trashedCollections.filter((item) => + neededTrashCollections.has(item.id), + ); + await localForage.setItem(DELETED_COLLECTION, filterCollections); +} + +async function getLastTrashSyncTime() { + return (await localForage.getItem(TRASH_TIME)) ?? 0; +} +export async function syncTrash( + collections: Collection[], + setTrashedFiles: (fs: EnteFile[]) => void, +): Promise { + const trash = await getLocalTrash(); + collections = [...collections, ...(await getLocalDeletedCollections())]; + const collectionMap = new Map( + collections.map((collection) => [collection.id, collection]), + ); + if (!getToken()) { + return; + } + const lastSyncTime = await getLastTrashSyncTime(); + + const updatedTrash = await updateTrash( + collectionMap, + lastSyncTime, + setTrashedFiles, + trash, + ); + await cleanTrashCollections(updatedTrash); +} + +export const updateTrash = async ( + collections: Map, + sinceTime: number, + setTrashedFiles: (fs: EnteFile[]) => void, + currentTrash: Trash, +): Promise => { + try { + let updatedTrash: Trash = [...currentTrash]; + let time = sinceTime; + + let resp; + do { + const token = getToken(); + if (!token) { + break; + } + resp = await HTTPService.get( + await apiURL("/trash/v2/diff"), + { + sinceTime: time, + }, + { + "X-Auth-Token": token, + }, + ); + // #Perf: This can be optimized by running the decryption in parallel + for (const trashItem of resp.data.diff as EncryptedTrashItem[]) { + const collectionID = trashItem.file.collectionID; + let collection = collections.get(collectionID); + if (!collection) { + collection = await getCollection(collectionID); + collections.set(collectionID, collection); + await localForage.setItem(DELETED_COLLECTION, [ + ...collections.values(), + ]); + } + if (!trashItem.isDeleted && !trashItem.isRestored) { + const decryptedFile = await decryptFile( + trashItem.file, + collection.key, + ); + updatedTrash.push({ ...trashItem, file: decryptedFile }); + } else { + updatedTrash = updatedTrash.filter( + (item) => item.file.id !== trashItem.file.id, + ); + } + } + + if (resp.data.diff.length) { + time = resp.data.diff.slice(-1)[0].updatedAt; + } + + setTrashedFiles(getTrashedFiles(updatedTrash)); + await localForage.setItem(TRASH, updatedTrash); + await localForage.setItem(TRASH_TIME, time); + } while (resp.data.hasMore); + return updatedTrash; + } catch (e) { + log.error("Get trash files failed", e); + } + return currentTrash; +}; + +export const emptyTrash = async () => { + try { + const token = getToken(); + if (!token) { + return; + } + const lastUpdatedAt = await getLastTrashSyncTime(); + + await HTTPService.post( + await apiURL("/trash/empty"), + { lastUpdatedAt }, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + null, + { + "X-Auth-Token": token, + }, + ); + } catch (e) { + log.error("empty trash failed", e); + throw e; + } +}; + +export const clearLocalTrash = async () => { + await localForage.setItem(TRASH, []); +}; From 94ce77c07b2aeac50dcf3c117cb5c11d7b900018 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:44:43 +0530 Subject: [PATCH 16/20] Move --- web/apps/photos/src/pages/gallery.tsx | 2 +- web/{apps/photos/src => packages/new/photos}/services/sync.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename web/{apps/photos/src => packages/new/photos}/services/sync.ts (98%) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 60ae8c915d..7b8d799930 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -52,6 +52,7 @@ import { } from "@/new/photos/services/search"; import type { SearchOption } from "@/new/photos/services/search/types"; import { initSettings } from "@/new/photos/services/settings"; +import { preCollectionsAndFilesSync, sync } from "@/new/photos/services/sync"; import { initUserDetailsOrTriggerSync, redirectToCustomerPortal, @@ -120,7 +121,6 @@ import { createUnCategorizedCollection, } from "services/collectionService"; import exportService from "services/export"; -import { preCollectionsAndFilesSync, sync } from "services/sync"; import uploadManager from "services/upload/uploadManager"; import { isTokenValid } from "services/userService"; import { diff --git a/web/apps/photos/src/services/sync.ts b/web/packages/new/photos/services/sync.ts similarity index 98% rename from web/apps/photos/src/services/sync.ts rename to web/packages/new/photos/services/sync.ts index 63bb1841fc..906eaad9b5 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/packages/new/photos/services/sync.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ import { isHiddenCollection } from "@/new/photos/services/collection"; import { getAllLatestCollections, From b09d6ab2a6c02fbf588f7281c9ce4430b50cf854 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:46:28 +0530 Subject: [PATCH 17/20] Sync after dedup --- web/packages/new/photos/services/dedup.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index e8942146be..6d6e59e485 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -11,6 +11,7 @@ import { } from "./collection"; import { getLocalCollections } from "./collections"; import { getLocalFiles } from "./files"; +import { syncFilesAndCollections } from "./sync"; /** * A group of duplicates as shown in the UI. @@ -270,7 +271,7 @@ export const removeSelectedDuplicateGroups = async ( } let np = 0; - const ntotal = filesToAdd.size + filesToTrash.length ? 1 : 0; + const ntotal = filesToAdd.size + filesToTrash.length ? 1 : 0 + /* sync */ 1; const tickProgress = () => onProgress((np++ / ntotal) * 100); // Process the adds. @@ -290,6 +291,9 @@ export const removeSelectedDuplicateGroups = async ( tickProgress(); } + await syncFilesAndCollections(); + tickProgress(); + return new Set(selectedDuplicateGroups.map((g) => g.id)); }; From 2845d7bfeb962a05a67bb6551a5eac166c68f386 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 18:49:34 +0530 Subject: [PATCH 18/20] lf --- .../photos/src/components/Collections/CollectionHeader.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/apps/photos/src/components/Collections/CollectionHeader.tsx b/web/apps/photos/src/components/Collections/CollectionHeader.tsx index 0e48f2da78..3a3fe8549c 100644 --- a/web/apps/photos/src/components/Collections/CollectionHeader.tsx +++ b/web/apps/photos/src/components/Collections/CollectionHeader.tsx @@ -21,6 +21,7 @@ import type { CollectionSummary, CollectionSummaryType, } from "@/new/photos/services/collection/ui"; +import { clearLocalTrash, emptyTrash } from "@/new/photos/services/collections"; import { isArchivedCollection, isPinnedCollection, @@ -50,10 +51,6 @@ import { GalleryContext } from "pages/gallery"; import React, { useCallback, useContext, useRef } from "react"; import { Trans } from "react-i18next"; import * as CollectionAPI from "services/collectionService"; -import { - emptyTrash, - clearLocalTrash, -} from "@/new/photos/services/collections"; import { SetFilesDownloadProgressAttributesCreator } from "types/gallery"; import { changeCollectionOrder, From b9c992cae02f641d03eb4b070397d29dafe0f8bd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 19:19:27 +0530 Subject: [PATCH 19/20] Dedup --- web/packages/new/photos/services/dedup.ts | 29 ++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index 6d6e59e485..f9750f2ac1 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -247,27 +247,34 @@ export const removeSelectedDuplicateGroups = async ( for (const duplicateGroup of selectedDuplicateGroups) { const retainedItem = duplicateGroupItemToRetain(duplicateGroup); + // Find the existing collection IDs to which this item already belongs. const existingCollectionIDs = new Set( retainedItem.collectionFiles.map((cf) => cf.collectionID), ); - // For each item, + + // For each item, find all the collections to which any of the files + // (except the file we're retaining) belongs. + const collectionIDs = new Set(); for (const item of duplicateGroup.items) { - // except the one we're retaining, + // Skip the item we're retaining, if (item.file.id == retainedItem.file.id) continue; - // Add the file we're retaining to each collection to which this - // item belongs. + // Determine the collections to which any of the item's files belong. for (const { collectionID } of item.collectionFiles) { - // Skip if already there - if (existingCollectionIDs.has(collectionID)) continue; - filesToAdd.set(collectionID, [ - ...(filesToAdd.get(collectionID) ?? []), - retainedItem.file, - ]); + if (!existingCollectionIDs.has(collectionID)) + collectionIDs.add(collectionID); } - // Add it to the list of items to be trashed. + // Add the item's files to list of collection files to be trashed. filesToTrash = filesToTrash.concat(item.collectionFiles); } + + // Add the file we're retaining to these (uniqued) collections. + for (const collectionID of collectionIDs) { + filesToAdd.set(collectionID, [ + ...(filesToAdd.get(collectionID) ?? []), + retainedItem.file, + ]); + } } let np = 0; From b71fa478b96a6d3fb9477cc56c455224b15363fa Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 26 Dec 2024 19:31:15 +0530 Subject: [PATCH 20/20] Remote expects uniques --- web/packages/new/photos/services/collection/index.ts | 10 +++++++--- web/packages/new/photos/services/dedup.ts | 9 ++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/web/packages/new/photos/services/collection/index.ts b/web/packages/new/photos/services/collection/index.ts index 33e602536c..24dc08b430 100644 --- a/web/packages/new/photos/services/collection/index.ts +++ b/web/packages/new/photos/services/collection/index.ts @@ -219,12 +219,16 @@ const encryptWithCollectionKey = async ( ); /** - * Make a remote request to move the given {@link collectionFiles} to trash. + * Make a remote request to move the given {@link files} to trash. + * + * @param files The {@link EnteFile}s to move to trash. The API request needs + * both a file ID and a collection ID, but there should be at most one entry for + * a particular fileID in this array. * * Does not modify local state. */ -export const moveToTrash = async (collectionFiles: EnteFile[]) => { - for (const batchFiles of batch(collectionFiles, requestBatchSize)) { +export const moveToTrash = async (files: EnteFile[]) => { + for (const batchFiles of batch(files, requestBatchSize)) { ensureOk( await fetch(await apiURL("/files/trash"), { method: "POST", diff --git a/web/packages/new/photos/services/dedup.ts b/web/packages/new/photos/services/dedup.ts index f9750f2ac1..5ceed26127 100644 --- a/web/packages/new/photos/services/dedup.ts +++ b/web/packages/new/photos/services/dedup.ts @@ -243,11 +243,10 @@ export const removeSelectedDuplicateGroups = async ( // const filesToAdd = new Map(); - let filesToTrash: EnteFile[] = []; + const filesToTrash: EnteFile[] = []; for (const duplicateGroup of selectedDuplicateGroups) { const retainedItem = duplicateGroupItemToRetain(duplicateGroup); - // Find the existing collection IDs to which this item already belongs. const existingCollectionIDs = new Set( retainedItem.collectionFiles.map((cf) => cf.collectionID), @@ -257,15 +256,15 @@ export const removeSelectedDuplicateGroups = async ( // (except the file we're retaining) belongs. const collectionIDs = new Set(); for (const item of duplicateGroup.items) { - // Skip the item we're retaining, + // Skip the item we're retaining. if (item.file.id == retainedItem.file.id) continue; // Determine the collections to which any of the item's files belong. for (const { collectionID } of item.collectionFiles) { if (!existingCollectionIDs.has(collectionID)) collectionIDs.add(collectionID); } - // Add the item's files to list of collection files to be trashed. - filesToTrash = filesToTrash.concat(item.collectionFiles); + // Move the item's file to trash. + filesToTrash.push(item.file); } // Add the file we're retaining to these (uniqued) collections.