TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] Batched visibility updates for file selections (#6315)

Browse Source
This commit is contained in:
Manav Rathi
2025-06-20 09:26:14 +05:30
committed by GitHub
parent a022d4aaa2 f743dc14d5
commit 8af2aa35d4
6 changed files with 189 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
web/apps/photos/src/utils/file/index.ts
View File

@@ -4,15 +4,9 @@ import log from "ente-base/log";
import { type Electron } from "ente-base/types/ipc";
import { saveAsFileAndRevokeObjectURL } from "ente-base/utils/web";
import { downloadManager } from "ente-gallery/services/download";
import { updateFileMagicMetadata } from "ente-gallery/services/file";
import { updateMagicMetadata } from "ente-gallery/services/magic-metadata";
import { detectFileTypeInfo } from "ente-gallery/utils/detect-type";
import { writeStream } from "ente-gallery/utils/native-stream";
import {
EnteFile,
FileMagicMetadataProps,
FileWithUpdatedMagicMetadata,
} from "ente-media/file";
import { EnteFile } from "ente-media/file";
import {
ItemVisibility,
fileFileName,
@@ -24,6 +18,7 @@ import {
deleteFromTrash,
moveToTrash,
} from "ente-new/photos/services/collection";
import { updateFilesVisibility } from "ente-new/photos/services/file";
import { safeFileName } from "ente-new/photos/utils/native-fs";
import { getData } from "ente-shared/storage/localStorage";
import { wait } from "ente-utils/promise";
@@ -65,28 +60,6 @@ export function getSelectedFiles(
return files.filter((file) => selectedFilesIDs.has(file.id));
}
export async function changeFilesVisibility(
files: EnteFile[],
visibility: ItemVisibility,
): Promise<EnteFile[]> {
const fileWithUpdatedMagicMetadataList: FileWithUpdatedMagicMetadata[] = [];
for (const file of files) {
const updatedMagicMetadataProps: FileMagicMetadataProps = {
visibility,
};
fileWithUpdatedMagicMetadataList.push({
file,
updatedMagicMetadata: await updateMagicMetadata(
updatedMagicMetadataProps,
file.magicMetadata,
file.key,
),
});
}
return await updateFileMagicMetadata(fileWithUpdatedMagicMetadataList);
}
export function isSharedFile(user: User, file: EnteFile) {
if (!user?.id || !file?.ownerID) {
return false;
@@ -387,10 +360,10 @@ export const handleFileOp = async (
await addMultipleToFavorites(files);
break;
case "archive":
await changeFilesVisibility(files, ItemVisibility.archived);
await updateFilesVisibility(files, ItemVisibility.archived);
break;
case "unarchive":
await changeFilesVisibility(files, ItemVisibility.visible);
await updateFilesVisibility(files, ItemVisibility.visible);
break;
case "hide":
try {

45
web/packages/gallery/services/file.ts
View File

@@ -9,7 +9,6 @@ import type {
EnteFile,
FilePublicMagicMetadata,
FilePublicMagicMetadataProps,
FileWithUpdatedMagicMetadata,
FileWithUpdatedPublicMagicMetadata,
} from "ente-media/file";
import { mergeMetadata } from "ente-media/file";
@@ -25,50 +24,6 @@ interface BulkUpdateMagicMetadataRequest {
metadataList: UpdateMagicMetadataRequest[];
}
export const updateFileMagicMetadata = async (
fileWithUpdatedMagicMetadataList: FileWithUpdatedMagicMetadata[],
) => {
const token = getToken();
if (!token) {
return;
}
const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] };
for (const {
file,
updatedMagicMetadata,
} of fileWithUpdatedMagicMetadataList) {
const { encryptedData, decryptionHeader } = await encryptMetadataJSON(
updatedMagicMetadata.data,
file.key,
);
reqBody.metadataList.push({
id: file.id,
magicMetadata: {
version: updatedMagicMetadata.version,
count: updatedMagicMetadata.count,
data: encryptedData,
header: decryptionHeader,
},
});
}
await HTTPService.put(
await apiURL("/files/magic-metadata"),
reqBody,
// @ts-ignore
null,
{ "X-Auth-Token": token },
);
return fileWithUpdatedMagicMetadataList.map(
({ file, updatedMagicMetadata }): EnteFile => ({
...file,
magicMetadata: {
...updatedMagicMetadata,
version: updatedMagicMetadata.version + 1,
},
}),
);
};
export const updateFilePublicMagicMetadata = async (
fileWithUpdatedPublicMagicMetadataList: FileWithUpdatedPublicMagicMetadata[],
): Promise<EnteFile[]> => {

31
web/packages/media/file-metadata.ts
View File

@@ -214,7 +214,7 @@ export const FileMetadata = z.looseObject({
* APIs refers to the (this) private metadata, even though the mutable public
* metadata is the much more frequently used of the two. See: [Note: Metadatum].
*/
export interface PrivateMagicMetadata {
export interface FilePrivateMagicMetadataData {
/**
* The visibility of the file.
*
@@ -222,10 +222,21 @@ export interface PrivateMagicMetadata {
* the private magic metadata. This allows the file's owner to share a file
* and independently edit its visibility without revealing their visibility
* preference to the other people with whom they have shared the file.
*
* Expected to be one of {@link ItemVisibility}.
*/
visibility?: ItemVisibility;
visibility?: number;
}
/**
* Zod schema for {@link FilePrivateMagicMetadataData}.
*
* See: [Note: Use looseObject for metadata Zod schemas]
*/
export const FilePrivateMagicMetadataData = z.looseObject({
visibility: z.number().nullish().transform(nullToUndefined),
});
/**
* The visibility of an Ente file or collection.
*/
@@ -259,8 +270,8 @@ export type ItemVisibility =
* - Unlike {@link FileMetadata}, this can change after the file has been
* uploaded.
*
* - Unlike {@link PrivateMagicMetadata}, this is available to all the people
* with whom the file has been shared.
* - Unlike {@link FilePrivateMagicMetadataData}, this is available to all the
* people with whom the file has been shared.
*
* For more details, see [Note: Metadatum].
*
@@ -421,9 +432,7 @@ export const filePrivateMagicMetadata = (file: EnteFile) => {
`Private magic metadata for ${fileLogID(file)} had not been decrypted even when the file reached the UI layer`,
);
}
// This cast is unavoidable in the current setup. We need to refactor the
// types so that this cast in not needed.
return file.magicMetadata.data as PrivateMagicMetadata;
return file.magicMetadata.data;
};
/**
@@ -564,10 +573,10 @@ export const fileCreationPhotoDate = (
* @param file The {@link EnteFile} whose public magic metadata we want to
* update.
*
* @param metadataUpdates A subset of {@link PrivateMagicMetadata} containing
* @param metadataUpdates A subset of {@link FilePrivateMagicMetadataData} containing
* the fields that we want to add or update.
*
* @returns An updated {@link PrivateMagicMetadata} object containing the
* @returns An updated {@link FilePrivateMagicMetadataData} object containing the
* (decrypted) metadata updates we just made. This is effectively what we would
* get if we to ask the remote for the latest file for this ID, except we don't
* do an actual sync and instead reconstruct it piecemeal.
@@ -606,7 +615,7 @@ export const fileCreationPhotoDate = (
*/
export const updateRemotePrivateMagicMetadata = async (
file: EnteFile,
metadataUpdates: Partial<PrivateMagicMetadata>,
metadataUpdates: Partial<FilePrivateMagicMetadataData>,
): Promise<FilePrivateMagicMetadata> => {
const existingMetadata = filePrivateMagicMetadata(file);
@@ -703,7 +712,7 @@ interface UpdateMagicMetadataRequest {
*/
const updateMagicMetadataRequest = async (
file: EnteFile,
metadata: PrivateMagicMetadata | PublicMagicMetadata,
metadata: FilePrivateMagicMetadataData | PublicMagicMetadata,
metadataVersion: number,
): Promise<UpdateMagicMetadataRequest> => {
// Drop all null or undefined values to obtain the syncable entries.

30
web/packages/media/file.ts
View File

@@ -8,7 +8,11 @@ import log from "ente-base/log";
import { nullishToBlank, nullToUndefined } from "ente-utils/transform";
import { z } from "zod/v4";
import { ignore } from "./collection";
import { fileFileName, FileMetadata, ItemVisibility } from "./file-metadata";
import {
fileFileName,
FileMetadata,
type FilePrivateMagicMetadataData,
} from "./file-metadata";
import { FileType } from "./file-type";
import { decryptMagicMetadata, RemoteMagicMetadata } from "./magic-metadata";
@@ -479,32 +483,14 @@ export const FileDiffResponse = z.object({
hasMore: z.boolean(),
});
export interface FileWithUpdatedMagicMetadata {
file: EnteFile;
updatedMagicMetadata: FileMagicMetadata;
}
export interface FileWithUpdatedPublicMagicMetadata {
file: EnteFile;
updatedPublicMagicMetadata: FilePublicMagicMetadata;
}
export interface FileMagicMetadataProps {
/**
* The visibility of the file
*
* The file's visibility is user specific attribute, and thus we keep it in
* the private magic metadata. This allows the file's owner to share a file
* and edit its visibility without making revealing their visibility
* preference to the people with whom they have shared the file.
*/
visibility?: ItemVisibility;
filePaths?: string[];
}
export type FileMagicMetadata = MagicMetadataCore<FileMagicMetadataProps>;
export type FileMagicMetadata = MagicMetadataCore<FilePrivateMagicMetadataData>;
export type FilePrivateMagicMetadata =
MagicMetadataCore<FileMagicMetadataProps>;
MagicMetadataCore<FilePrivateMagicMetadataData>;
export interface FilePublicMagicMetadataProps {
/**
@@ -743,7 +729,7 @@ export const decryptRemoteFile = async (
key,
);
// TODO(RE):
const data = genericMM.data as FileMagicMetadataProps;
const data = genericMM.data as FilePrivateMagicMetadataData;
// TODO(RE):
magicMetadata = { ...genericMM, header: "", data };
}

49
web/packages/new/photos/services/collection.ts
View File

@@ -9,6 +9,7 @@ import {
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import { apiURL } from "ente-base/origins";
import { ensureMasterKeyFromSession } from "ente-base/session";
import type { UpdateMagicMetadataRequest } from "ente-gallery/services/file";
import {
CollectionSubType,
decryptRemoteCollection,
@@ -28,23 +29,12 @@ import { ItemVisibility } from "ente-media/file-metadata";
import {
createMagicMetadata,
encryptMagicMetadata,
type RemoteMagicMetadata,
} from "ente-media/magic-metadata";
import { batch } from "ente-utils/array";
import { z } from "zod/v4";
import { requestBatchSize } from "./file";
import { ensureUserKeyPair, getPublicKey } from "./user";
/**
* An reasonable but otherwise arbitrary number of items (e.g. files) to include
* in a single API request.
*
* Remote will reject too big payloads, and requests which affect multiple items
* (e.g. files when moving files to a collection) are expected to be batched to
* keep each request of a reasonable size. By default, we break the request into
* batches of 1000.
*/
const requestBatchSize = 1000;
const uncategorizedCollectionName = "Uncategorized";
const defaultHiddenCollectionName = ".hidden";
export const defaultHiddenCollectionUserFacingName = "Hidden";
@@ -541,7 +531,7 @@ export const updateCollectionSortOrder = async (
) => updateCollectionPublicMagicMetadata(collection, { asc });
/**
* Update the private magic metadata contents of a collection on remote.
* Update the private magic metadata of a collection on remote.
*
* Remote only, does not modify local state.
*
@@ -557,7 +547,7 @@ export const updateCollectionSortOrder = async (
*
* See: [Note: Magic metadata data cannot have nullish values]
*/
export const updateCollectionPrivateMagicMetadata = async (
const updateCollectionPrivateMagicMetadata = async (
{ id, key, magicMetadata }: Collection,
updates: CollectionPrivateMagicMetadataData,
) =>
@@ -572,32 +562,11 @@ export const updateCollectionPrivateMagicMetadata = async (
),
});
/**
* The payload of the remote requests for updating the magic metadata of a
* single collection.
*/
interface UpdateCollectionMagicMetadataRequest {
/**
* Collection ID
*/
id: number;
/**
* The updated magic metadata.
*
* Remote usually enforces the following constraints when we're trying to
* update already existing data.
*
* - The version should be same as the existing version.
* - The count should be greater than or equal to the existing count.
*/
magicMetadata: RemoteMagicMetadata;
}
/**
* Update the private magic metadata of a single collection on remote.
*/
const putCollectionsMagicMetadata = async (
updateRequest: UpdateCollectionMagicMetadataRequest,
updateRequest: UpdateMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/collections/magic-metadata"), {
@@ -608,7 +577,7 @@ const putCollectionsMagicMetadata = async (
);
/**
* Update the public magic metadata contents of a collection on remote.
* Update the public magic metadata of a collection on remote.
*
* Remote only, does not modify local state.
*
@@ -634,7 +603,7 @@ const updateCollectionPublicMagicMetadata = async (
* Update the public magic metadata of a single collection on remote.
*/
const putCollectionsPublicMagicMetadata = async (
updateRequest: UpdateCollectionMagicMetadataRequest,
updateRequest: UpdateMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/collections/public-magic-metadata"), {
@@ -645,7 +614,7 @@ const putCollectionsPublicMagicMetadata = async (
);
/**
* Update the per-sharee magic metadata contents of a collection on remote.
* Update the per-sharee magic metadata of a collection on remote.
*
* Remote only, does not modify local state.
*
@@ -671,7 +640,7 @@ const updateCollectionShareeMagicMetadata = async (
* Update the sharee magic metadata of a single shared collection on remote.
*/
const putCollectionsShareeMagicMetadata = async (
updateRequest: UpdateCollectionMagicMetadataRequest,
updateRequest: UpdateMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/collections/sharee-magic-metadata"), {

148
web/packages/new/photos/services/file.ts Normal file
View File

@@ -0,0 +1,148 @@
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import { apiURL } from "ente-base/origins";
import type { EnteFile, EnteFile2 } from "ente-media/file";
import type {
FilePrivateMagicMetadataData,
ItemVisibility,
} from "ente-media/file-metadata";
import {
createMagicMetadata,
encryptMagicMetadata,
type RemoteMagicMetadata,
} from "ente-media/magic-metadata";
import { batch } from "ente-utils/array";
/**
* An reasonable but otherwise arbitrary number of items (e.g. files) to include
* in a single API request.
*
* Remote will reject too big payloads, and requests which affect multiple items
* (e.g. files when moving files to a collection, changing the visibility of
* selected files) are expected to be batched to keep each request of a
* reasonable size. By default, we break the request into batches of 1000.
*/
export const requestBatchSize = 1000;
/**
* Perform an operation on batches, concurrently.
*
* The given {@link items} are split into batches, each of
* {@link requestBatchSize}. The provided operation is called on all these
* batches, in parallel, by using `Promise.all`. When all the operations are
* complete, the function returns with an array of results (one from each batch
* promise resolution).
*
* @param items The arbitrary items to break into {@link requestBatchSize}
* batches.
*
* @param op The operation to perform on each batch.
*
* @returns An array of results, one from each batch operation. For details,
* including behaviour on errors, see `Promise.all`.
*/
export const performInBatches = <T, U>(
items: T[],
op: (batchItems: T[]) => Promise<U>,
): Promise<U[]> => Promise.all(batch(items, requestBatchSize).map(op));
/**
* Change the visibility (normal, archived, hidden) of a list of files on
* remote.
*
* Remote only, does not modify local state.
*
* @param files The list of files whose visibility we want to change. All the
* files will get their visibility updated to the new, provided, value.
*
* @param visibility The new visibility (normal, archived, hidden).
*/
export const updateFilesVisibility = async (
files: EnteFile[],
visibility: ItemVisibility,
) =>
performInBatches(files, (b) =>
updateFilesPrivateMagicMetadata(b, { visibility }),
);
/**
* Update the private magic metadata of a list of files on remote.
*
* Remote only, does not modify local state.
*
* @param file The list of files whose magic metadata we want to update. The
* same updates will be applied to the magic metadata of all the files.
*
* The existing magic metadata of the provided files is used both to obtain the
* current magic metadata version, and the existing contents on top of which the
* updates are applied, so it is imperative that both these values are up to
* sync with remote otherwise the update will fail.
*
* @param updates A non-empty subset of {@link FilePrivateMagicMetadataData}
* entries.
*
* See: [Note: Magic metadata data cannot have nullish values]
*/
const updateFilesPrivateMagicMetadata = async (
files: EnteFile2[],
updates: FilePrivateMagicMetadataData,
) =>
putFilesMagicMetadata({
metadataList: await Promise.all(
files.map(async ({ id, key, magicMetadata }) => ({
id,
magicMetadata: await encryptMagicMetadata(
createMagicMetadata(
{ ...magicMetadata?.data, ...updates },
magicMetadata?.version,
),
key,
),
})),
),
});
/**
* The payload of the remote requests for updating the magic metadata of a
* single item (file or collection).
*/
export interface UpdateMagicMetadataRequest {
/**
* File or collection ID
*/
id: number;
/**
* The updated magic metadata.
*
* Remote usually enforces the following constraints when we're trying to
* update already existing data.
*
* - The version should be same as the existing version.
* - The count should be greater than or equal to the existing count.
*/
magicMetadata: RemoteMagicMetadata;
}
/**
* The payload of the remote requests for updating the magic metadata of a
* multiple items.
*
* Currently this is only used by endpoints that update magic metadata for a
* list of files.
*/
export interface UpdateMultipleMagicMetadataRequest {
metadataList: UpdateMagicMetadataRequest[];
}
/**
* Update the private magic metadata of a list of files on remote.
*/
const putFilesMagicMetadata = async (
updateRequest: UpdateMultipleMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/files/magic-metadata"), {
method: "PUT",
headers: await authenticatedRequestHeaders(),
body: JSON.stringify(updateRequest),
}),
);