diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 540e2e6306..7f64d7d9d0 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -4,15 +4,9 @@ import log from "ente-base/log"; import { type Electron } from "ente-base/types/ipc"; import { saveAsFileAndRevokeObjectURL } from "ente-base/utils/web"; import { downloadManager } from "ente-gallery/services/download"; -import { updateFileMagicMetadata } from "ente-gallery/services/file"; -import { updateMagicMetadata } from "ente-gallery/services/magic-metadata"; import { detectFileTypeInfo } from "ente-gallery/utils/detect-type"; import { writeStream } from "ente-gallery/utils/native-stream"; -import { - EnteFile, - FileMagicMetadataProps, - FileWithUpdatedMagicMetadata, -} from "ente-media/file"; +import { EnteFile } from "ente-media/file"; import { ItemVisibility, fileFileName, @@ -24,6 +18,7 @@ import { deleteFromTrash, moveToTrash, } from "ente-new/photos/services/collection"; +import { updateFilesVisibility } from "ente-new/photos/services/file"; import { safeFileName } from "ente-new/photos/utils/native-fs"; import { getData } from "ente-shared/storage/localStorage"; import { wait } from "ente-utils/promise"; @@ -65,28 +60,6 @@ export function getSelectedFiles( return files.filter((file) => selectedFilesIDs.has(file.id)); } -export async function changeFilesVisibility( - files: EnteFile[], - visibility: ItemVisibility, -): Promise { - const fileWithUpdatedMagicMetadataList: FileWithUpdatedMagicMetadata[] = []; - for (const file of files) { - const updatedMagicMetadataProps: FileMagicMetadataProps = { - visibility, - }; - - fileWithUpdatedMagicMetadataList.push({ - file, - updatedMagicMetadata: await updateMagicMetadata( - updatedMagicMetadataProps, - file.magicMetadata, - file.key, - ), - }); - } - return await updateFileMagicMetadata(fileWithUpdatedMagicMetadataList); -} - export function isSharedFile(user: User, file: EnteFile) { if (!user?.id || !file?.ownerID) { return false; @@ -387,10 +360,10 @@ export const handleFileOp = async ( await addMultipleToFavorites(files); break; case "archive": - await changeFilesVisibility(files, ItemVisibility.archived); + await updateFilesVisibility(files, ItemVisibility.archived); break; case "unarchive": - await changeFilesVisibility(files, ItemVisibility.visible); + await updateFilesVisibility(files, ItemVisibility.visible); break; case "hide": try { diff --git a/web/packages/gallery/services/file.ts b/web/packages/gallery/services/file.ts index 39dd207d01..c83c059cb9 100644 --- a/web/packages/gallery/services/file.ts +++ b/web/packages/gallery/services/file.ts @@ -9,7 +9,6 @@ import type { EnteFile, FilePublicMagicMetadata, FilePublicMagicMetadataProps, - FileWithUpdatedMagicMetadata, FileWithUpdatedPublicMagicMetadata, } from "ente-media/file"; import { mergeMetadata } from "ente-media/file"; @@ -25,50 +24,6 @@ interface BulkUpdateMagicMetadataRequest { metadataList: UpdateMagicMetadataRequest[]; } -export const updateFileMagicMetadata = async ( - fileWithUpdatedMagicMetadataList: FileWithUpdatedMagicMetadata[], -) => { - const token = getToken(); - if (!token) { - return; - } - const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] }; - for (const { - file, - updatedMagicMetadata, - } of fileWithUpdatedMagicMetadataList) { - const { encryptedData, decryptionHeader } = await encryptMetadataJSON( - updatedMagicMetadata.data, - file.key, - ); - reqBody.metadataList.push({ - id: file.id, - magicMetadata: { - version: updatedMagicMetadata.version, - count: updatedMagicMetadata.count, - data: encryptedData, - header: decryptionHeader, - }, - }); - } - await HTTPService.put( - await apiURL("/files/magic-metadata"), - reqBody, - // @ts-ignore - null, - { "X-Auth-Token": token }, - ); - return fileWithUpdatedMagicMetadataList.map( - ({ file, updatedMagicMetadata }): EnteFile => ({ - ...file, - magicMetadata: { - ...updatedMagicMetadata, - version: updatedMagicMetadata.version + 1, - }, - }), - ); -}; - export const updateFilePublicMagicMetadata = async ( fileWithUpdatedPublicMagicMetadataList: FileWithUpdatedPublicMagicMetadata[], ): Promise => { diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index d351bad6cb..542303d7ec 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -214,7 +214,7 @@ export const FileMetadata = z.looseObject({ * APIs refers to the (this) private metadata, even though the mutable public * metadata is the much more frequently used of the two. See: [Note: Metadatum]. */ -export interface PrivateMagicMetadata { +export interface FilePrivateMagicMetadataData { /** * The visibility of the file. * @@ -222,10 +222,21 @@ export interface PrivateMagicMetadata { * the private magic metadata. This allows the file's owner to share a file * and independently edit its visibility without revealing their visibility * preference to the other people with whom they have shared the file. + * + * Expected to be one of {@link ItemVisibility}. */ - visibility?: ItemVisibility; + visibility?: number; } +/** + * Zod schema for {@link FilePrivateMagicMetadataData}. + * + * See: [Note: Use looseObject for metadata Zod schemas] + */ +export const FilePrivateMagicMetadataData = z.looseObject({ + visibility: z.number().nullish().transform(nullToUndefined), +}); + /** * The visibility of an Ente file or collection. */ @@ -259,8 +270,8 @@ export type ItemVisibility = * - Unlike {@link FileMetadata}, this can change after the file has been * uploaded. * - * - Unlike {@link PrivateMagicMetadata}, this is available to all the people - * with whom the file has been shared. + * - Unlike {@link FilePrivateMagicMetadataData}, this is available to all the + * people with whom the file has been shared. * * For more details, see [Note: Metadatum]. * @@ -421,9 +432,7 @@ export const filePrivateMagicMetadata = (file: EnteFile) => { `Private magic metadata for ${fileLogID(file)} had not been decrypted even when the file reached the UI layer`, ); } - // This cast is unavoidable in the current setup. We need to refactor the - // types so that this cast in not needed. - return file.magicMetadata.data as PrivateMagicMetadata; + return file.magicMetadata.data; }; /** @@ -564,10 +573,10 @@ export const fileCreationPhotoDate = ( * @param file The {@link EnteFile} whose public magic metadata we want to * update. * - * @param metadataUpdates A subset of {@link PrivateMagicMetadata} containing + * @param metadataUpdates A subset of {@link FilePrivateMagicMetadataData} containing * the fields that we want to add or update. * - * @returns An updated {@link PrivateMagicMetadata} object containing the + * @returns An updated {@link FilePrivateMagicMetadataData} object containing the * (decrypted) metadata updates we just made. This is effectively what we would * get if we to ask the remote for the latest file for this ID, except we don't * do an actual sync and instead reconstruct it piecemeal. @@ -606,7 +615,7 @@ export const fileCreationPhotoDate = ( */ export const updateRemotePrivateMagicMetadata = async ( file: EnteFile, - metadataUpdates: Partial, + metadataUpdates: Partial, ): Promise => { const existingMetadata = filePrivateMagicMetadata(file); @@ -703,7 +712,7 @@ interface UpdateMagicMetadataRequest { */ const updateMagicMetadataRequest = async ( file: EnteFile, - metadata: PrivateMagicMetadata | PublicMagicMetadata, + metadata: FilePrivateMagicMetadataData | PublicMagicMetadata, metadataVersion: number, ): Promise => { // Drop all null or undefined values to obtain the syncable entries. diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts index 0e76b30bd9..e07c2645ea 100644 --- a/web/packages/media/file.ts +++ b/web/packages/media/file.ts @@ -8,7 +8,11 @@ import log from "ente-base/log"; import { nullishToBlank, nullToUndefined } from "ente-utils/transform"; import { z } from "zod/v4"; import { ignore } from "./collection"; -import { fileFileName, FileMetadata, ItemVisibility } from "./file-metadata"; +import { + fileFileName, + FileMetadata, + type FilePrivateMagicMetadataData, +} from "./file-metadata"; import { FileType } from "./file-type"; import { decryptMagicMetadata, RemoteMagicMetadata } from "./magic-metadata"; @@ -479,32 +483,14 @@ export const FileDiffResponse = z.object({ hasMore: z.boolean(), }); -export interface FileWithUpdatedMagicMetadata { - file: EnteFile; - updatedMagicMetadata: FileMagicMetadata; -} - export interface FileWithUpdatedPublicMagicMetadata { file: EnteFile; updatedPublicMagicMetadata: FilePublicMagicMetadata; } -export interface FileMagicMetadataProps { - /** - * The visibility of the file - * - * The file's visibility is user specific attribute, and thus we keep it in - * the private magic metadata. This allows the file's owner to share a file - * and edit its visibility without making revealing their visibility - * preference to the people with whom they have shared the file. - */ - visibility?: ItemVisibility; - filePaths?: string[]; -} - -export type FileMagicMetadata = MagicMetadataCore; +export type FileMagicMetadata = MagicMetadataCore; export type FilePrivateMagicMetadata = - MagicMetadataCore; + MagicMetadataCore; export interface FilePublicMagicMetadataProps { /** @@ -743,7 +729,7 @@ export const decryptRemoteFile = async ( key, ); // TODO(RE): - const data = genericMM.data as FileMagicMetadataProps; + const data = genericMM.data as FilePrivateMagicMetadataData; // TODO(RE): magicMetadata = { ...genericMM, header: "", data }; } diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index 2b91e367fb..a78c78f4f0 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -9,6 +9,7 @@ import { import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; import { apiURL } from "ente-base/origins"; import { ensureMasterKeyFromSession } from "ente-base/session"; +import type { UpdateMagicMetadataRequest } from "ente-gallery/services/file"; import { CollectionSubType, decryptRemoteCollection, @@ -28,23 +29,12 @@ import { ItemVisibility } from "ente-media/file-metadata"; import { createMagicMetadata, encryptMagicMetadata, - type RemoteMagicMetadata, } from "ente-media/magic-metadata"; import { batch } from "ente-utils/array"; import { z } from "zod/v4"; +import { requestBatchSize } from "./file"; import { ensureUserKeyPair, getPublicKey } from "./user"; -/** - * An reasonable but otherwise arbitrary number of items (e.g. files) to include - * in a single API request. - * - * Remote will reject too big payloads, and requests which affect multiple items - * (e.g. files when moving files to a collection) are expected to be batched to - * keep each request of a reasonable size. By default, we break the request into - * batches of 1000. - */ -const requestBatchSize = 1000; - const uncategorizedCollectionName = "Uncategorized"; const defaultHiddenCollectionName = ".hidden"; export const defaultHiddenCollectionUserFacingName = "Hidden"; @@ -541,7 +531,7 @@ export const updateCollectionSortOrder = async ( ) => updateCollectionPublicMagicMetadata(collection, { asc }); /** - * Update the private magic metadata contents of a collection on remote. + * Update the private magic metadata of a collection on remote. * * Remote only, does not modify local state. * @@ -557,7 +547,7 @@ export const updateCollectionSortOrder = async ( * * See: [Note: Magic metadata data cannot have nullish values] */ -export const updateCollectionPrivateMagicMetadata = async ( +const updateCollectionPrivateMagicMetadata = async ( { id, key, magicMetadata }: Collection, updates: CollectionPrivateMagicMetadataData, ) => @@ -572,32 +562,11 @@ export const updateCollectionPrivateMagicMetadata = async ( ), }); -/** - * The payload of the remote requests for updating the magic metadata of a - * single collection. - */ -interface UpdateCollectionMagicMetadataRequest { - /** - * Collection ID - */ - id: number; - /** - * The updated magic metadata. - * - * Remote usually enforces the following constraints when we're trying to - * update already existing data. - * - * - The version should be same as the existing version. - * - The count should be greater than or equal to the existing count. - */ - magicMetadata: RemoteMagicMetadata; -} - /** * Update the private magic metadata of a single collection on remote. */ const putCollectionsMagicMetadata = async ( - updateRequest: UpdateCollectionMagicMetadataRequest, + updateRequest: UpdateMagicMetadataRequest, ) => ensureOk( await fetch(await apiURL("/collections/magic-metadata"), { @@ -608,7 +577,7 @@ const putCollectionsMagicMetadata = async ( ); /** - * Update the public magic metadata contents of a collection on remote. + * Update the public magic metadata of a collection on remote. * * Remote only, does not modify local state. * @@ -634,7 +603,7 @@ const updateCollectionPublicMagicMetadata = async ( * Update the public magic metadata of a single collection on remote. */ const putCollectionsPublicMagicMetadata = async ( - updateRequest: UpdateCollectionMagicMetadataRequest, + updateRequest: UpdateMagicMetadataRequest, ) => ensureOk( await fetch(await apiURL("/collections/public-magic-metadata"), { @@ -645,7 +614,7 @@ const putCollectionsPublicMagicMetadata = async ( ); /** - * Update the per-sharee magic metadata contents of a collection on remote. + * Update the per-sharee magic metadata of a collection on remote. * * Remote only, does not modify local state. * @@ -671,7 +640,7 @@ const updateCollectionShareeMagicMetadata = async ( * Update the sharee magic metadata of a single shared collection on remote. */ const putCollectionsShareeMagicMetadata = async ( - updateRequest: UpdateCollectionMagicMetadataRequest, + updateRequest: UpdateMagicMetadataRequest, ) => ensureOk( await fetch(await apiURL("/collections/sharee-magic-metadata"), { diff --git a/web/packages/new/photos/services/file.ts b/web/packages/new/photos/services/file.ts new file mode 100644 index 0000000000..e9c3b2cbbb --- /dev/null +++ b/web/packages/new/photos/services/file.ts @@ -0,0 +1,148 @@ +import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; +import { apiURL } from "ente-base/origins"; +import type { EnteFile, EnteFile2 } from "ente-media/file"; +import type { + FilePrivateMagicMetadataData, + ItemVisibility, +} from "ente-media/file-metadata"; +import { + createMagicMetadata, + encryptMagicMetadata, + type RemoteMagicMetadata, +} from "ente-media/magic-metadata"; +import { batch } from "ente-utils/array"; + +/** + * An reasonable but otherwise arbitrary number of items (e.g. files) to include + * in a single API request. + * + * Remote will reject too big payloads, and requests which affect multiple items + * (e.g. files when moving files to a collection, changing the visibility of + * selected files) are expected to be batched to keep each request of a + * reasonable size. By default, we break the request into batches of 1000. + */ +export const requestBatchSize = 1000; + +/** + * Perform an operation on batches, concurrently. + * + * The given {@link items} are split into batches, each of + * {@link requestBatchSize}. The provided operation is called on all these + * batches, in parallel, by using `Promise.all`. When all the operations are + * complete, the function returns with an array of results (one from each batch + * promise resolution). + * + * @param items The arbitrary items to break into {@link requestBatchSize} + * batches. + * + * @param op The operation to perform on each batch. + * + * @returns An array of results, one from each batch operation. For details, + * including behaviour on errors, see `Promise.all`. + */ +export const performInBatches = ( + items: T[], + op: (batchItems: T[]) => Promise, +): Promise => Promise.all(batch(items, requestBatchSize).map(op)); + +/** + * Change the visibility (normal, archived, hidden) of a list of files on + * remote. + * + * Remote only, does not modify local state. + * + * @param files The list of files whose visibility we want to change. All the + * files will get their visibility updated to the new, provided, value. + * + * @param visibility The new visibility (normal, archived, hidden). + */ +export const updateFilesVisibility = async ( + files: EnteFile[], + visibility: ItemVisibility, +) => + performInBatches(files, (b) => + updateFilesPrivateMagicMetadata(b, { visibility }), + ); + +/** + * Update the private magic metadata of a list of files on remote. + * + * Remote only, does not modify local state. + * + * @param file The list of files whose magic metadata we want to update. The + * same updates will be applied to the magic metadata of all the files. + * + * The existing magic metadata of the provided files is used both to obtain the + * current magic metadata version, and the existing contents on top of which the + * updates are applied, so it is imperative that both these values are up to + * sync with remote otherwise the update will fail. + * + * @param updates A non-empty subset of {@link FilePrivateMagicMetadataData} + * entries. + * + * See: [Note: Magic metadata data cannot have nullish values] + */ +const updateFilesPrivateMagicMetadata = async ( + files: EnteFile2[], + updates: FilePrivateMagicMetadataData, +) => + putFilesMagicMetadata({ + metadataList: await Promise.all( + files.map(async ({ id, key, magicMetadata }) => ({ + id, + magicMetadata: await encryptMagicMetadata( + createMagicMetadata( + { ...magicMetadata?.data, ...updates }, + magicMetadata?.version, + ), + key, + ), + })), + ), + }); + +/** + * The payload of the remote requests for updating the magic metadata of a + * single item (file or collection). + */ +export interface UpdateMagicMetadataRequest { + /** + * File or collection ID + */ + id: number; + /** + * The updated magic metadata. + * + * Remote usually enforces the following constraints when we're trying to + * update already existing data. + * + * - The version should be same as the existing version. + * - The count should be greater than or equal to the existing count. + */ + magicMetadata: RemoteMagicMetadata; +} + +/** + * The payload of the remote requests for updating the magic metadata of a + * multiple items. + * + * Currently this is only used by endpoints that update magic metadata for a + * list of files. + */ +export interface UpdateMultipleMagicMetadataRequest { + metadataList: UpdateMagicMetadataRequest[]; +} + +/** + * Update the private magic metadata of a list of files on remote. + */ +const putFilesMagicMetadata = async ( + updateRequest: UpdateMultipleMagicMetadataRequest, +) => + ensureOk( + await fetch(await apiURL("/files/magic-metadata"), { + method: "PUT", + headers: await authenticatedRequestHeaders(), + body: JSON.stringify(updateRequest), + }), + );