From fe224b5ab2800a00030dbb2424202da38f72f8d0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 11:43:01 +0530 Subject: [PATCH 01/11] dec --- web/packages/media/magic-metadata.ts | 37 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/web/packages/media/magic-metadata.ts b/web/packages/media/magic-metadata.ts index 801ff914e2..813f119bcf 100644 --- a/web/packages/media/magic-metadata.ts +++ b/web/packages/media/magic-metadata.ts @@ -1,4 +1,5 @@ -import { encryptMetadataJSON } from "ente-base/crypto"; +import { decryptMetadataJSON, encryptMetadataJSON } from "ente-base/crypto"; +import type { BytesOrB64 } from "ente-base/crypto/types"; import { z } from "zod/v4"; /** @@ -120,8 +121,8 @@ export interface MagicMetadata { * discarded before obtaining the final object that will be encrypted (and whose * count will be used). * - * @param key The key (as a base64 string) to use for encrypting the - * {@link data} contents of {@link magicMetadata}. + * @param key The key to use for encrypting the {@link data} contents of + * {@link magicMetadata}. * * The specific key used depends on the object whose metadata this is. For * example, if this were the {@link pubMagicMetadata} associated with an @@ -136,7 +137,7 @@ export interface MagicMetadata { */ export const encryptMagicMetadata = async ( magicMetadata: MagicMetadata | undefined, - key: string, + key: BytesOrB64, ): Promise => { if (!magicMetadata) return undefined; @@ -155,3 +156,31 @@ export const encryptMagicMetadata = async ( return { version, count, data, header }; }; + +/** + * Decrypt the magic metadata received from remote into an object suitable for + * use and persistence by us (the client). + * + * This is meant as the inverse of {@link encryptMagicMetadata}, see that + * function's documentation for more information. + */ +export const decryptMagicMetadata = async ( + remoteMagicMetadata: RemoteMagicMetadata | undefined, + key: BytesOrB64, +): Promise | undefined> => { + if (!remoteMagicMetadata) return undefined; + + const { + version, + count, + data: encryptedData, + header: decryptionHeader, + } = remoteMagicMetadata; + + const data = await decryptMetadataJSON( + { encryptedData, decryptionHeader }, + key, + ); + + return { version, count, data }; +}; From 00aabfc283776b7f6729f628290092a1f012f6b7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 12:01:03 +0530 Subject: [PATCH 02/11] Separate concerns --- web/packages/media/collection.ts | 99 +++++++++++++++++++--------- web/packages/media/magic-metadata.ts | 24 ++++--- 2 files changed, 83 insertions(+), 40 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index f466e65f3b..cac7c3f25c 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -117,8 +117,27 @@ export interface CollectionUser { /** * Zod schema for {@link CollectionUser}. + * + * [Note: Enums in remote objects] + * + * In some cases remote returns a value which is part of a known set. The + * underlying type might be an integer or a string. + * + * While Zod allows us to validate enum types during a parse, we refrain from + * doing this as this would cause existing clients to break if remote were to in + * the future add new enum cases. + * + * This is especially pertinent for objects which we might persist locally, + * since the client code which persists the object might not know about a + * particular enum case but a future client which reads the saved value might. + * + * So we keep the underlying data type (string or number) as it is instead of + * converting / validating to an enum or discarding unknown values. + * + * As an example consider the {@link role} item in {@link RemoteCollectionUser}. + * There is a known set of values ({@link CollectionParticipantRole}) this can + * be, but in the Zod schema we keep the data type as a string. */ -// TODO: Use me. export const RemoteCollectionUser = z.object({ id: z.number(), email: z.string().nullish(), @@ -130,6 +149,8 @@ type RemoteCollectionUser = z.infer; /** * Zod schema for {@link Collection}. * + * See: [Note: Schema suffix for exported Zod schemas]. + * * TODO(RE): The following reasoning is not fully correct, since we anyways need * to do a conversion when decrypting the collection's fields. The intent is to * update this once we've figured out something that also works with the current @@ -140,35 +161,13 @@ type RemoteCollectionUser = z.infer; * Objects like files and collections that we get from remote are treated * specially when it comes to schema validation. * - * 1. Enum conversion. - * 2. Loose objects. - * 3. Blank handling. * 4. Casting instead of validating when reading local values. * * Let us take a concrete example of the {@link Collection} TypeScript type, * whose zod schema is defined by {@link RemoteCollection}. * - * The collection that we get from remote contains (nested) enum types - a - * {@link CollectionParticipantRole}. While zod allows us to validate enum types - * during a parse, this would cause existing clients to break if remote were to - * in the future add new enum cases. So when parsing we'd like to keep the role - * value as a string. - * - * This is especially important for a object like {@link Collection} which is - * also persisted locally, because a current client code might persist a object - * which might be read by future client code that understands more fields. So we - * use zod's {@link looseObject} directive on the {@link RemoteCollection} to - * ensure we don't discard fields we don't recognize, in a manner similar to - * [Note: Use looseObject for metadata Zod schemas]. - * - * In keeping with this general principle of retaining the object we get from - * remote as vintage as possible, we also don't do transformations to deal with - * various remote idiosyncracies. For example, for the role field remote (in - * some cases) uses blanks to indicate missing values. While zod would allow to - * transform these, we just let it be as remote had sent it. - * - * Finally, while we always use the {@link RemoteCollection} schema validator - * when parsing remote network responses, we don't do the same when reading the + * While we always use the {@link RemoteCollection} schema validator when + * parsing remote network responses, we don't do the same when reading the * persisted values. This is to retain the performance characteristics of the * existing code. This might seem miniscule for the {@link Collection} example, * but users can easily have hundreds of thousands of {@link EnteFile}s @@ -179,11 +178,6 @@ type RemoteCollectionUser = z.infer; * So when reading arrays of these objects from local DB, we do a cast instead * of do a runtime zod validation. * - * To summarize, for certain remote objects which are also persisted to disk in - * potentially large numbers, we (a) try to retain the remote semantics as much - * as possible to avoid the need for a parsing / transform step, and (b) we - * don't even do a parsing / transform step when reading them from local DB. - * * This means that the "types might lie". On the other hand, we need to special * case only very specific objects this way: * @@ -192,8 +186,7 @@ type RemoteCollectionUser = z.infer; * 3. {@link Trash} * */ -// TODO: Use me -export const RemoteCollection = z.object({ +export const RemoteCollectionSchema = z.object({ id: z.number(), owner: RemoteCollectionUser, encryptedKey: z.string(), @@ -235,6 +228,8 @@ export const RemoteCollection = z.object({ RemoteMagicMetadataSchema.nullish().transform(nullToUndefined), }); +export type RemoteCollection = z.infer; + export interface EncryptedCollection { /** * The collection's globally unique ID. @@ -361,6 +356,46 @@ export interface PublicURL { memLimit?: number; } +/** + * Pertinent information about the Ente user on whose behalf we are trying to + * decrypt a collection received from remote. + */ +interface CollectionDecryptionUser { + /** + * The ID of the currently logged in user. + */ + userID: number; + /** + * The base64 encoded master key of the currently logged in user. + * + * This is used for decrypting collections that the user owns. + */ + masterKey: string; + /** + * The base64 encoded private key of the currently logged in user. + * + * This is used for decrypting collections that have been shared with the + * user. + */ + privateKey: string; +} + +/** + * Decrypt a collection obtained from remote ({@link RemoteCollection}) into the + * collection object that we use and persist on the client ({@link Collection}). + * + * @param remoteCollection The collection object we received from remote. This + * can be thought of as an envelope, the actual consumables within it are + * encrypted using the user's master key or public key. + * + * @param user The ID of the currently logged in user, and their keys. This is + * needed to decrypt the encrypted remote collection. + */ +export const decryptRemoteCollection = ( + remoteCollection: RemoteCollection, + user: CollectionDecryptionUser, +) => {}; + export interface UpdatePublicURL { collectionID: number; disablePassword?: boolean; diff --git a/web/packages/media/magic-metadata.ts b/web/packages/media/magic-metadata.ts index 813f119bcf..81d8fc3c62 100644 --- a/web/packages/media/magic-metadata.ts +++ b/web/packages/media/magic-metadata.ts @@ -1,5 +1,4 @@ import { decryptMetadataJSON, encryptMetadataJSON } from "ente-base/crypto"; -import type { BytesOrB64 } from "ente-base/crypto/types"; import { z } from "zod/v4"; /** @@ -7,9 +6,18 @@ import { z } from "zod/v4"; * remote. It is effectively an envelope of the encrypted metadata contents with * some bookkeeping information attached. * - * See {@link RemoteMagicMetadata} for the corresponding type. Since this module - * exports both the Zod schema and the TypeScript type, to avoid confusion the - * schema is suffixed by "Schema". + * See {@link RemoteMagicMetadata} for the corresponding type. + * + * [Note: Schema suffix for exported Zod schemas] + * + * Since this module exports both the Zod schema and the TypeScript type, to + * avoid confusion the schema is suffixed by "Schema". + * + * This is a general pattern we follow in all cases where we need to export the + * Zod schema so that it can be used as a composition block when defining other + * schemas. These are meant to be exceptions though, whenever possible, the + * schema should be internal to the module (and can thus have the same name as + * the TypeScript type, without the need for a disambiguating suffix). */ export const RemoteMagicMetadataSchema = z.object({ version: z.number(), @@ -121,8 +129,8 @@ export interface MagicMetadata { * discarded before obtaining the final object that will be encrypted (and whose * count will be used). * - * @param key The key to use for encrypting the {@link data} contents of - * {@link magicMetadata}. + * @param key The base64 encoded key to use for encrypting the {@link data} + * contents of {@link magicMetadata}. * * The specific key used depends on the object whose metadata this is. For * example, if this were the {@link pubMagicMetadata} associated with an @@ -137,7 +145,7 @@ export interface MagicMetadata { */ export const encryptMagicMetadata = async ( magicMetadata: MagicMetadata | undefined, - key: BytesOrB64, + key: string, ): Promise => { if (!magicMetadata) return undefined; @@ -166,7 +174,7 @@ export const encryptMagicMetadata = async ( */ export const decryptMagicMetadata = async ( remoteMagicMetadata: RemoteMagicMetadata | undefined, - key: BytesOrB64, + key: string, ): Promise | undefined> => { if (!remoteMagicMetadata) return undefined; From 228bf55a8d679a5c18b77ed49dfbaa1670627564 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 12:29:36 +0530 Subject: [PATCH 03/11] C2 --- web/packages/media/collection.ts | 123 +++++++++++++--------- web/packages/media/file.ts | 4 +- web/packages/new/photos/services/files.ts | 11 +- 3 files changed, 81 insertions(+), 57 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index cac7c3f25c..a59141214a 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -43,7 +43,7 @@ import { RemoteMagicMetadataSchema } from "./magic-metadata"; * * In the remote schema, each item ({@link EnteFile}) is always associated * with a collection. The same item may belong to multiple collections (See: - * [Note: Collection File]), but it must belong to at least one collection. + * [Note: Collection file]), but it must belong to at least one collection. * * In some scenarios, e.g. when deleting the last collection to which a file * belongs, the file would thus get orphaned and violate the schema @@ -231,6 +231,57 @@ export const RemoteCollectionSchema = z.object({ export type RemoteCollection = z.infer; export interface EncryptedCollection { + id: number; + owner: CollectionUser; + encryptedKey: string; + keyDecryptionNonce: string; + name?: string; + encryptedName: string; + nameDecryptionNonce: string; + type: CollectionType; + sharees: CollectionUser[]; + publicURLs?: PublicURL[]; + updationTime: number; + isDeleted: boolean; + magicMetadata: EncryptedMagicMetadata; + pubMagicMetadata: EncryptedMagicMetadata; + sharedMagicMetadata: EncryptedMagicMetadata; +} + +export interface Collection + extends Omit< + EncryptedCollection, + | "encryptedKey" + | "keyDecryptionNonce" + | "encryptedName" + | "nameDecryptionNonce" + | "magicMetadata" + | "pubMagicMetadata" + | "sharedMagicMetadata" + > { + key: string; + name: string; + magicMetadata: CollectionMagicMetadata; + pubMagicMetadata: CollectionPublicMagicMetadata; + sharedMagicMetadata: CollectionShareeMagicMetadata; +} + +/** + * A collection, as used and persisted locally by the client. + * + * A collection is roughly equivalent to an "album", though there can be special + * type of collections (like "favorites") which have special behaviours attached + * to them. + * + * A collection contains zero or more files ({@link EnteFile}). + * + * A collection can be owned by the user (in whose context this code is + * running), or might be a collection that is shared with them. + * + * TODO: This type supercedes {@link Collection}. Once migration is done, rename + * this to drop the "2" suffix. + */ +export interface Collection2 { /** * The collection's globally unique ID. * @@ -253,34 +304,36 @@ export interface EncryptedCollection { * - {@link role} will be blank. */ owner: CollectionUser; - encryptedKey: string; - keyDecryptionNonce: string; - name?: string; - encryptedName: string; - nameDecryptionNonce: string; + /** + * The "collection key" (base64 encoded). + * + * The collection key is used to encrypt and decrypt that files that are + * associated with the collection. See: [Note: Collection file]. + */ + key: string; + /** + * The name of the collection. + */ + name: string; /** * The type of the collection. * - * See the documentation of {@link CollectionType} for more details. + * Expected to be one of {@link CollectionType}. */ - type: CollectionType; - /** - * TODO(RE): Remove me? - */ - attributes: collectionAttributes; + type: string; // CollectionType; /** * The other Ente users with whom the collection has been shared with. * * Within the {@link CollectionUser} instances of the {@link sharee} field: * * - {@link email} will be set. - * - {@link role} will be one of "VIEWER" or "COLLABORATOR". + * - {@link role} is expected to be one of "VIEWER" or "COLLABORATOR". */ - sharees: CollectionUser[]; + sharees?: CollectionUser[]; /** - * Public links for the collection. + * Public links that can be used to access and update the collection. */ - publicURLs?: PublicURL[]; + publicURLs?: unknown; // PublicURL[]; /** * The last time the collection was updated (epoch microseconds). * @@ -290,45 +343,20 @@ export interface EncryptedCollection { * - When the collection's own fields are modified. */ updationTime: number; - isDeleted: boolean; - magicMetadata: EncryptedMagicMetadata; - pubMagicMetadata: EncryptedMagicMetadata; - sharedMagicMetadata: EncryptedMagicMetadata; -} - -export interface Collection - extends Omit< - EncryptedCollection, - | "encryptedKey" - | "keyDecryptionNonce" - | "encryptedName" - | "nameDecryptionNonce" - | "magicMetadata" - | "pubMagicMetadata" - | "sharedMagicMetadata" - > { - /** - * The "collection key" (base64 encoded). - */ - key: string; - /** - * The name of the collection. - */ - name: string; /** * Mutable metadata associated with the collection that is only visible to * the owner of the collection. * * See: [Note: Metadatum] */ - magicMetadata: CollectionMagicMetadata; + magicMetadata?: unknown; //CollectionMagicMetadata; /** * Public mutable metadata associated with the collection that is visible to * all users with whom the collection has been shared. * * See: [Note: Metadatum] */ - pubMagicMetadata: CollectionPublicMagicMetadata; + pubMagicMetadata?: unknown; //CollectionPublicMagicMetadata; /** * Private mutable metadata associated with the collection that is only * visible to the current user, if they're not the owner. @@ -341,7 +369,7 @@ export interface Collection * * See: [Note: Metadatum] */ - sharedMagicMetadata: CollectionShareeMagicMetadata; + sharedMagicMetadata?: unknown; // CollectionShareeMagicMetadata; } export interface PublicURL { @@ -394,7 +422,7 @@ interface CollectionDecryptionUser { export const decryptRemoteCollection = ( remoteCollection: RemoteCollection, user: CollectionDecryptionUser, -) => {}; +): Promise => {}; export interface UpdatePublicURL { collectionID: number; @@ -415,11 +443,6 @@ export interface CreatePublicAccessTokenRequest { deviceLimit?: number; } -export interface collectionAttributes { - encryptedPath?: string; - pathDecryptionNonce?: string; -} - export interface RemoveFromCollectionRequest { collectionID: number; fileIDs: number[]; diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts index c5d8937e58..9315ce233d 100644 --- a/web/packages/media/file.ts +++ b/web/packages/media/file.ts @@ -79,7 +79,7 @@ export interface EncryptedEnteFile { * * The same file (ID) may be associated with multiple collectionID, in which * case there will be multiple {@link EnteFile} entries for each - * ({@link id}, {@link collectionID}) pair. See: [Note: Collection File]. + * ({@link id}, {@link collectionID}) pair. See: [Note: Collection file]. */ collectionID: number; /** @@ -152,7 +152,7 @@ export interface EncryptedEnteFile { * While the file ID is unique, we'd can still have multiple entries for each * file ID in our local state, one per collection IDs to which the file belongs. * That is, the uniqueness is across the (fileID, collectionID) pairs. See - * [Note: Collection File]. + * [Note: Collection file]. */ export interface EnteFile extends Omit< diff --git a/web/packages/new/photos/services/files.ts b/web/packages/new/photos/services/files.ts index 56752ee480..fadbb67c9a 100644 --- a/web/packages/new/photos/services/files.ts +++ b/web/packages/new/photos/services/files.ts @@ -188,13 +188,14 @@ export const sortFiles = (files: EnteFile[], sortAsc = false) => { }; /** - * [Note: Collection File] + * [Note: Collection file] * * File IDs themselves are unique across all the files for the user (in fact, - * they're unique across all the files in an Ente instance). However, we still - * can have multiple entries for the same file ID in our local database because - * the unit of account is not actually a file, but a "Collection File": a - * collection and file pair. + * they're unique across all the files in an Ente instance). + * + * However, we can have multiple entries for the same file ID in our local + * database and/or remote responses because the unit of account is not file, but + * a "Collection File" – a collection and file pair. * * For example, if the same file is symlinked into two collections, then we will * have two "Collection File" entries for it, both with the same file ID, but From 49b4adc8437c74f1558b989f31230ef9c13b20b1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 13:29:14 +0530 Subject: [PATCH 04/11] Test a suffix --- web/packages/media/collection.ts | 1 + web/packages/media/magic-metadata.ts | 123 ++++++++++++------ .../new/photos/services/collection.ts | 16 +++ 3 files changed, 97 insertions(+), 43 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index a59141214a..977ec602be 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -239,6 +239,7 @@ export interface EncryptedCollection { encryptedName: string; nameDecryptionNonce: string; type: CollectionType; + attributes: unknown; sharees: CollectionUser[]; publicURLs?: PublicURL[]; updationTime: number; diff --git a/web/packages/media/magic-metadata.ts b/web/packages/media/magic-metadata.ts index 81d8fc3c62..fc934026f3 100644 --- a/web/packages/media/magic-metadata.ts +++ b/web/packages/media/magic-metadata.ts @@ -3,8 +3,9 @@ import { z } from "zod/v4"; /** * Zod schema of the mutable metadatum objects that we send to or receive from - * remote. It is effectively an envelope of the encrypted metadata contents with - * some bookkeeping information attached. + * remote. Even though it is named so, it is not the metadata itself, but an + * envelope containing the encrypted metadata contents with some bookkeeping + * information attached. * * See {@link RemoteMagicMetadata} for the corresponding type. * @@ -14,10 +15,10 @@ import { z } from "zod/v4"; * avoid confusion the schema is suffixed by "Schema". * * This is a general pattern we follow in all cases where we need to export the - * Zod schema so that it can be used as a composition block when defining other - * schemas. These are meant to be exceptions though, whenever possible, the - * schema should be internal to the module (and can thus have the same name as - * the TypeScript type, without the need for a disambiguating suffix). + * Zod schema so other modules can compose schema definitions. Such exports are + * meant to be exceptions though; usually the schema should be internal to the + * module (and so can have the same name as the TypeScript type without the need + * for a disambiguating suffix). */ export const RemoteMagicMetadataSchema = z.object({ version: z.number(), @@ -27,16 +28,16 @@ export const RemoteMagicMetadataSchema = z.object({ }); /** - * Any of the mutable metadata fields, as represented by remote. + * The type of the mutable metadata fields, as represented by remote. * * See: [Note: Metadatum] * - * This is the encrypted magic metadata that we send to and receive from remote. + * This is the magic metadata envelope that we send to and receive from remote. * It contains the encrypted contents, and a version + count of fields that help * ensure that clients do not overwrite updates with stale objects. * - * When decrypt these into specific {@link MagicMetadata} instantiations before - * using and storing them on the client. + * When decrypt these into specific local {@link MagicMetadataEnvelope} + * instantiations before using and storing them on the client. * * See {@link RemoteMagicMetadataSchema} for the corresponding Zod schema. The * same structure is used to store the metadata for various kinds of objects @@ -70,10 +71,10 @@ export interface RemoteMagicMetadata { * This are the base64 encoded bytes of the encrypted magic metadata JSON * object. * - * The encryption will happen using the "key" of the object whose magic - * metadata field this is. For example, if this is the + * The encryption will happen using the "key" associated with the object + * whose magic metadata field this is. For example, if this is the * {@link pubMagicMetadata} of an {@link EnteFile}, then the file's key will - * be used for encryption. + * be used for encryption and decryption. */ data: string; /** @@ -88,8 +89,8 @@ export interface RemoteMagicMetadata { * by the client. * * The bookkeeping fields ({@link version} and {@link count}) are copied over - * from the envelope ({@link RemoteMagicMetadata}), while the encrypted contents - * ({@link data} and {@link header}) are decrypted into a JSON object. + * from the remote envelope ({@link RemoteMagicMetadata}), while the encrypted + * contents ({@link data} and {@link header}) are decrypted into a JSON object. * * Since different types of magic metadata fields have different JSON contents, * so this decrypted JSON object is parameterized as `T` in this generic type. @@ -105,69 +106,105 @@ export interface RemoteMagicMetadata { * sense - to describe the shape of any of the magic metadata like fields used * in various types. See: [Note: Metadatum]. */ -export interface MagicMetadata { +export interface MagicMetadataEnvelope { version: number; count: number; + /** + * The "metadata" itself. + * + * This is expected to be a JSON object, whose exact schema depends on the + * field for which this envelope is being used. + */ data: T; } /** - * Encrypt the provided magic metadata into a form that can be used in + * Encrypt the provided magic metadata envelope into a form that can be used in * communication with remote, updating the count if needed. * - * @param magicMetadata The decrypted {@link MagicMetadata} that is being used - * by the client. + * @param envelope The decrypted {@link MagicMetadataEnvelope} that + * is being used by the client. * * As a usability convenience, this function allows passing `undefined` as the - * {@link magicMetadata}. In such a case, it will return `undefined` too. + * {@link envelope}. In such cases, it will return `undefined` too. * * It is okay for the count in the envelope to be out of sync with the count in * the actual JSON object contents, this function will update the envelope count * with the number of entries in the JSON object. * - * Any entries in {@link magicMetadata} that are `null` or `undefined` are - * discarded before obtaining the final object that will be encrypted (and whose - * count will be used). + * Any entries in {@link envelope} that are `null` or `undefined` + * are discarded before obtaining the final object that will be encrypted (and + * whose count will be used). * * @param key The base64 encoded key to use for encrypting the {@link data} - * contents of {@link magicMetadata}. + * contents of {@link envelope}. * * The specific key used depends on the object whose metadata this is. For * example, if this were the {@link pubMagicMetadata} associated with an * {@link EnteFile}, then this would be the file's key. * * @returns a {@link RemoteMagicMetadata} object that contains the encrypted - * contents of the {@link data} present in the provided {@link magicMetadata}. - * The {@link version} is copied over from the provided {@link magicMetadata}, - * while the {@link count} is obtained from the number of entries in the JSON - * object (the {@link data} property of {@link magicMetadata}). + * contents of the {@link data} present in the provided + * {@link envelope}. The {@link version} is copied over from the + * provided {@link envelope}, while the {@link count} is obtained + * from the number of entries in the trimmed JSON object obtained from the + * {@link data} property of {@link envelope}. * */ -export const encryptMagicMetadata = async ( - magicMetadata: MagicMetadata | undefined, +export const encryptMagicMetadata = async ( + envelope: MagicMetadataEnvelope | undefined, key: string, ): Promise => { - if (!magicMetadata) return undefined; + if (!envelope) return undefined; - // Discard any entries that are `null` or `undefined`. - const jsonObject = Object.fromEntries( - Object.entries(magicMetadata.data ?? {}).filter( - ([, v]) => v !== null && v !== undefined, - ), - ); + const { version } = envelope; - const version = magicMetadata.version; - const count = Object.keys(jsonObject).length; + const newEnvelope = createMagicMetadataEnvelope(envelope.data); + const { count } = newEnvelope; const { encryptedData: data, decryptionHeader: header } = - await encryptMetadataJSON(jsonObject, key); + await encryptMetadataJSON(newEnvelope.data, key); return { version, count, data, header }; }; /** - * Decrypt the magic metadata received from remote into an object suitable for - * use and persistence by us (the client). + * A function to wrap an arbitrary JSON object in an envelope expected of + * various magic metadata fields. + * + * A trimmed JSON object is obtained from the provided JSON object by removing + * any entries whose values is `undefined` or `null`. + * + * Then, + * + * - The `version` is set to 0. + * - The `count` is set to the number of entries in the trimmed JSON object. + * - The `data` is set to the trimmed JSON object. + * + * {@link encryptMagicMetadata} internally uses this function to obtain an + * up-to-date envelope before computing the count and encrypting the contents. + * This function is also exported for use in places where we don't have an + * existing envelope, and would like to create one from scratch. + * + * @param data The "metadata" JSON object. Since TypeScript does not have a JSON + * type, this uses the `unknown` type. + */ +export const createMagicMetadataEnvelope = (data: unknown) => { + // Discard any entries that are `undefined` or `null`. + const jsonObject = Object.fromEntries( + Object.entries(data ?? {}).filter( + ([, v]) => v !== undefined && v !== null, + ), + ); + + const count = Object.keys(jsonObject).length; + + return { version: 0, count, data: jsonObject }; +}; + +/** + * Decrypt the magic metadata (envelope) received from remote into an object + * suitable for use and persistence by the client. * * This is meant as the inverse of {@link encryptMagicMetadata}, see that * function's documentation for more information. @@ -175,7 +212,7 @@ export const encryptMagicMetadata = async ( export const decryptMagicMetadata = async ( remoteMagicMetadata: RemoteMagicMetadata | undefined, key: string, -): Promise | undefined> => { +): Promise => { if (!remoteMagicMetadata) return undefined; const { diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index dcbe72b9b0..da2d4b647b 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -81,6 +81,22 @@ export const getCollectionUserFacingName = (collection: Collection) => { return collection.name; }; +/** + * Create a new collection on remote, and return its local representation. + * + * Remote only, does not modify local state. + * + * @param name The name of the new collection. + * + * @param type The type of the new collection. + * + * @param magicMetadata Optional metadata to use as the collection's private + * mutable metadata when creating the new collection. + */ +export const createCollection => { + +} + /** * Return a map of the (user-facing) collection name, indexed by collection ID. */ From 432d44f4af0d1ac04f84e185954b6858b14df0f9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 13:40:53 +0530 Subject: [PATCH 05/11] Sketch --- web/packages/media/collection.ts | 54 +++++++++++++++---- web/packages/media/file-metadata.ts | 12 +++-- .../new/photos/services/collection.ts | 10 ++-- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 977ec602be..16cd248b1e 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -425,6 +425,51 @@ export const decryptRemoteCollection = ( user: CollectionDecryptionUser, ): Promise => {}; +/** + * Additional context stored as part of the collection's magic metadata to + * augment the {@link type} associated with a {@link Collection}. + */ +export const CollectionSubType = { + default: 0, + defaultHidden: 1, + quicklink: 2, +} as const; + +export type CollectionSubType = + (typeof CollectionSubType)[keyof typeof CollectionSubType]; + +/** + * Mutable private metadata associated with an {@link Collection}. + * + * - Unlike {@link CollectionPublicMagicMetadata} this is only available to the + * owner of the file. + * + * See: [Note: Private magic metadata is called magic metadata on remote] + */ +export interface CollectionPrivateMagicMetadata { + /** + * The (owner specific) visibility of the collection. + * + * and independently edit its visibility without revealing their visibility + * preference to the other people with whom they have shared the file. + */ + visibility?: ItemVisibility; + /** + * The {@link CollectionSubType}, if applicable. + */ + subType?: CollectionSubType; + /** + * An overrride to the sort ordering used for the collection. + * + * - For pinned collections, this will be set to `1`. Pinned collections + * will be moved to the beginning of the sort order. + * + * - Otherwise, the collection is a normal (unpinned) collection, and will + * retain its natural sort position. + */ + order?: number; +} + export interface UpdatePublicURL { collectionID: number; disablePassword?: boolean; @@ -449,15 +494,6 @@ export interface RemoveFromCollectionRequest { fileIDs: number[]; } -export const CollectionSubType = { - default: 0, - defaultHidden: 1, - quicklink: 2, -} as const; - -export type CollectionSubType = - (typeof CollectionSubType)[keyof typeof CollectionSubType]; - export interface CollectionMagicMetadataProps { visibility?: ItemVisibility; subType?: CollectionSubType; diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 4c2e25bf12..9c1c3f5f82 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -200,11 +200,17 @@ export interface PrivateMagicMetadata { * The visibility of an Ente file or collection. */ export const ItemVisibility = { - /** The normal state - The item is visible. */ + /** + * The normal state - The item is visible. + */ visible: 0, - /** The item has been archived. */ + /** + * The item has been archived. + */ archived: 1, - /** The item has been hidden. */ + /** + * The item has been hidden. + */ hidden: 2, } as const; diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index da2d4b647b..6697850801 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -6,6 +6,8 @@ import { CollectionSubType, type Collection, type CollectionNewParticipantRole, + type CollectionPrivateMagicMetadata, + type CollectionType, } from "ente-media/collection"; import { type EnteFile } from "ente-media/file"; import { ItemVisibility } from "ente-media/file-metadata"; @@ -93,9 +95,11 @@ export const getCollectionUserFacingName = (collection: Collection) => { * @param magicMetadata Optional metadata to use as the collection's private * mutable metadata when creating the new collection. */ -export const createCollection => { - -} +export const createCollection = ( + name: string, + type: CollectionType, + magicMetadata?: CollectionPrivateMagicMetadata, +): Promise => {}; /** * Return a map of the (user-facing) collection name, indexed by collection ID. From 8585f3881c267cd86529ff0c3d29536619e5295e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 13:52:36 +0530 Subject: [PATCH 06/11] Sketch 2 --- web/packages/media/collection.ts | 22 ++++++++ .../new/photos/services/collection.ts | 56 ++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 16cd248b1e..6faa204e41 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -425,6 +425,28 @@ export const decryptRemoteCollection = ( user: CollectionDecryptionUser, ): Promise => {}; +/** + * Decrypt a remote collection using the provided key. + * + * This is the lower level implementation of {@link decryptRemoteCollection} for + * use by code that already has determined which key should be used for + * decrypting the collection. + * + * @param remoteCollection The collection to decrypt. + * + * @param key The base64 encoded key to use for decrypting the various encrypted + * fields in {@link remoteCollection}. + * + * @returns A decrypted collection ({@link Collection2}). + */ +export const decryptRemoteCollectionUsingKey = ( + remoteCollection: RemoteCollection, + key: string, +): Promise => { + // eslint-disable-next-line @typescript-eslint/no-base-to-string + throw new Error("TODO" + remoteCollection.toString() + key.toString()); +}; + /** * Additional context stored as part of the collection's magic metadata to * augment the {@link type} associated with a {@link Collection}. diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index 6697850801..cda3f50bb7 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -1,17 +1,27 @@ import type { User } from "ente-accounts/services/user"; -import { boxSeal, encryptBox } from "ente-base/crypto"; +import { boxSeal, encryptBox, generateKey } from "ente-base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; import { apiURL } from "ente-base/origins"; +import { ensureMasterKeyFromSession } from "ente-base/session"; import { CollectionSubType, + decryptRemoteCollectionUsingKey, + RemoteCollectionSchema, type Collection, + type Collection2, type CollectionNewParticipantRole, type CollectionPrivateMagicMetadata, type CollectionType, + type RemoteCollection, } from "ente-media/collection"; import { type EnteFile } from "ente-media/file"; import { ItemVisibility } from "ente-media/file-metadata"; +import { + createMagicMetadataEnvelope, + encryptMagicMetadata, +} from "ente-media/magic-metadata"; import { batch } from "ente-utils/array"; +import { z } from "zod/v4"; import { getPublicKey } from "./user"; /** @@ -95,11 +105,51 @@ export const getCollectionUserFacingName = (collection: Collection) => { * @param magicMetadata Optional metadata to use as the collection's private * mutable metadata when creating the new collection. */ -export const createCollection = ( +export const createCollection2 = async ( name: string, type: CollectionType, magicMetadata?: CollectionPrivateMagicMetadata, -): Promise => {}; +): Promise => { + const masterKey = await ensureMasterKeyFromSession(); + const collectionKey = await generateKey(); + const { encryptedData: encryptedKey, nonce: keyDecryptionNonce } = + await encryptBox(collectionKey, masterKey); + const { encryptedData: encryptedName, nonce: nameDecryptionNonce } = + await encryptBox(new TextEncoder().encode(name), collectionKey); + const remoteMagicMetadata = await encryptMagicMetadata( + createMagicMetadataEnvelope(magicMetadata), + collectionKey, + ); + + const remoteCollection = await postCollections({ + encryptedKey, + keyDecryptionNonce, + encryptedName, + nameDecryptionNonce, + type, + ...(remoteMagicMetadata ? { magicMetadata: remoteMagicMetadata } : {}), + }); + + return decryptRemoteCollectionUsingKey(remoteCollection, masterKey); +}; + +/** + * Create a collection on remote with the provided data, and return the new + * remote collection object returned by remote on success. + */ +const postCollections = async ( + collectionData: Partial, +): Promise => { + const res = await fetch(await apiURL("/collections"), { + method: "POST", + headers: await authenticatedRequestHeaders(), + body: JSON.stringify(collectionData), + }); + ensureOk(res); + return z + .object({ collection: RemoteCollectionSchema }) + .parse(await res.json()).collection; +}; /** * Return a map of the (user-facing) collection name, indexed by collection ID. From 23609c4bb9a3236fd32136aeb29c2bf9efb120d5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 14:34:34 +0530 Subject: [PATCH 07/11] Revert the envelope terminology --- web/packages/media/collection.ts | 6 +- web/packages/media/magic-metadata.ts | 107 +++++++++--------- .../new/photos/services/collection.ts | 19 ++-- 3 files changed, 65 insertions(+), 67 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 6faa204e41..466a666c48 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -463,12 +463,12 @@ export type CollectionSubType = /** * Mutable private metadata associated with an {@link Collection}. * - * - Unlike {@link CollectionPublicMagicMetadata} this is only available to the - * owner of the file. + * - Unlike {@link CollectionPublicMagicMetadataData} this is only available to + * the owner of the file. * * See: [Note: Private magic metadata is called magic metadata on remote] */ -export interface CollectionPrivateMagicMetadata { +export interface CollectionPrivateMagicMetadataData { /** * The (owner specific) visibility of the collection. * diff --git a/web/packages/media/magic-metadata.ts b/web/packages/media/magic-metadata.ts index fc934026f3..3f043aac15 100644 --- a/web/packages/media/magic-metadata.ts +++ b/web/packages/media/magic-metadata.ts @@ -3,8 +3,7 @@ import { z } from "zod/v4"; /** * Zod schema of the mutable metadatum objects that we send to or receive from - * remote. Even though it is named so, it is not the metadata itself, but an - * envelope containing the encrypted metadata contents with some bookkeeping + * remote. It contains an encrypted metadata JSON object with some bookkeeping * information attached. * * See {@link RemoteMagicMetadata} for the corresponding type. @@ -32,12 +31,12 @@ export const RemoteMagicMetadataSchema = z.object({ * * See: [Note: Metadatum] * - * This is the magic metadata envelope that we send to and receive from remote. - * It contains the encrypted contents, and a version + count of fields that help - * ensure that clients do not overwrite updates with stale objects. + * This is the magic metadata that we send to and receive from remote. It + * contains an encrypted JSON object, and a version + count of entries to ensure + * that clients do not overwrite already applied updates with stale objects. * - * When decrypt these into specific local {@link MagicMetadataEnvelope} - * instantiations before using and storing them on the client. + * When decrypt these into specific local {@link MagicMetadata} instantiations + * before using and storing them on the client. * * See {@link RemoteMagicMetadataSchema} for the corresponding Zod schema. The * same structure is used to store the metadata for various kinds of objects @@ -89,11 +88,13 @@ export interface RemoteMagicMetadata { * by the client. * * The bookkeeping fields ({@link version} and {@link count}) are copied over - * from the remote envelope ({@link RemoteMagicMetadata}), while the encrypted - * contents ({@link data} and {@link header}) are decrypted into a JSON object. + * from the remote object ({@link RemoteMagicMetadata}), while the encrypted + * contents ({@link data} and {@link header} of {@link RemoteMagicMetadata}) are + * decrypted into a JSON object that is stored as {@link data}. * - * Since different types of magic metadata fields have different JSON contents, - * so this decrypted JSON object is parameterized as `T` in this generic type. + * Since different types of magic metadata fields are expected to contain + * different JSON objects as {@link data}, the type is generic, with the generic + * parameter `T` standing for the type of the JSON object. * * > Since TypeScript does not currently have a native JSON type, this T doesn't * > have other constraints enforced by the type system, but it is meant to be a @@ -106,71 +107,64 @@ export interface RemoteMagicMetadata { * sense - to describe the shape of any of the magic metadata like fields used * in various types. See: [Note: Metadatum]. */ -export interface MagicMetadataEnvelope { +export interface MagicMetadata { version: number; count: number; /** * The "metadata" itself. * * This is expected to be a JSON object, whose exact schema depends on the - * field for which this envelope is being used. + * magic metadata field in the parent object. */ data: T; } /** - * Encrypt the provided magic metadata envelope into a form that can be used in - * communication with remote, updating the count if needed. + * Encrypt the provided magic metadata into a form that can be used in + * communication with remote, trimming the JSON object and updating the count. * - * @param envelope The decrypted {@link MagicMetadataEnvelope} that - * is being used by the client. + * @param magicMetadata The decrypted {@link MagicMetadata} that is being used + * by the client. * - * As a usability convenience, this function allows passing `undefined` as the - * {@link envelope}. In such cases, it will return `undefined` too. + * Any entries in {@link magicMetadata}'s {@link data} that are `null` or + * `undefined` are discarded before obtaining the final object that will be + * encrypted (and whose count will be used). * - * It is okay for the count in the envelope to be out of sync with the count in - * the actual JSON object contents, this function will update the envelope count - * with the number of entries in the JSON object. - * - * Any entries in {@link envelope} that are `null` or `undefined` - * are discarded before obtaining the final object that will be encrypted (and - * whose count will be used). + * So in particular, it is okay for the count in the {@link magicMetadata} to be + * out of sync with the count in the `data` JSON object because this function + * will recompute the count using the trimmed JSON object. * * @param key The base64 encoded key to use for encrypting the {@link data} - * contents of {@link envelope}. + * contents of {@link magicMetadata}. * - * The specific key used depends on the object whose metadata this is. For - * example, if this were the {@link pubMagicMetadata} associated with an - * {@link EnteFile}, then this would be the file's key. + * The specific key used depends on the object whose magic metadata field this + * is meant for. For example, if this were the {@link pubMagicMetadata} + * associated with an {@link EnteFile}, then this would be the file's key. * * @returns a {@link RemoteMagicMetadata} object that contains the encrypted - * contents of the {@link data} present in the provided - * {@link envelope}. The {@link version} is copied over from the - * provided {@link envelope}, while the {@link count} is obtained - * from the number of entries in the trimmed JSON object obtained from the - * {@link data} property of {@link envelope}. - * + * contents of the {@link data} present in the provided {@link magicMetadata}. + * The {@link version} is copied over from the provided {@link magicMetadata}, + * while the {@link count} is obtained from the number of entries in the trimmed + * JSON object that was encrypted. */ export const encryptMagicMetadata = async ( - envelope: MagicMetadataEnvelope | undefined, + magicMetadata: MagicMetadata, key: string, ): Promise => { - if (!envelope) return undefined; + const { version } = magicMetadata; - const { version } = envelope; - - const newEnvelope = createMagicMetadataEnvelope(envelope.data); - const { count } = newEnvelope; + const newMM = createMagicMetadata(magicMetadata.data); + const { count } = newMM; const { encryptedData: data, decryptionHeader: header } = - await encryptMetadataJSON(newEnvelope.data, key); + await encryptMetadataJSON(newMM.data, key); return { version, count, data, header }; }; /** - * A function to wrap an arbitrary JSON object in an envelope expected of - * various magic metadata fields. + * A function to wrap an arbitrary JSON object in the {@link MagicMetadata} + * envelope used for the various magic metadata fields. * * A trimmed JSON object is obtained from the provided JSON object by removing * any entries whose values is `undefined` or `null`. @@ -182,14 +176,14 @@ export const encryptMagicMetadata = async ( * - The `data` is set to the trimmed JSON object. * * {@link encryptMagicMetadata} internally uses this function to obtain an - * up-to-date envelope before computing the count and encrypting the contents. - * This function is also exported for use in places where we don't have an - * existing envelope, and would like to create one from scratch. + * trimmed `data` and its corresponding `count`. This function is also exported + * for use in places where we just have a JSON object and would like to a new + * {@link MagicMetadata} object to house it from scratch. * - * @param data The "metadata" JSON object. Since TypeScript does not have a JSON - * type, this uses the `unknown` type. + * @param data A JSON object. Since TypeScript does not have a JSON type, this + * uses the `unknown` type. */ -export const createMagicMetadataEnvelope = (data: unknown) => { +export const createMagicMetadata = (data: unknown) => { // Discard any entries that are `undefined` or `null`. const jsonObject = Object.fromEntries( Object.entries(data ?? {}).filter( @@ -203,16 +197,19 @@ export const createMagicMetadataEnvelope = (data: unknown) => { }; /** - * Decrypt the magic metadata (envelope) received from remote into an object - * suitable for use and persistence by the client. + * Decrypt the magic metadata received from remote into an object suitable for + * use and persistence by the client. * * This is meant as the inverse of {@link encryptMagicMetadata}, see that - * function's documentation for more information. + * function's documentation for details. + * + * As a usability convenience, this function allows passing `undefined` as the + * {@link remoteMagicMetadata}. In such cases, it will return `undefined` too. */ export const decryptMagicMetadata = async ( remoteMagicMetadata: RemoteMagicMetadata | undefined, key: string, -): Promise => { +): Promise => { if (!remoteMagicMetadata) return undefined; const { diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index cda3f50bb7..2755369673 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -17,7 +17,7 @@ import { import { type EnteFile } from "ente-media/file"; import { ItemVisibility } from "ente-media/file-metadata"; import { - createMagicMetadataEnvelope, + createMagicMetadata, encryptMagicMetadata, } from "ente-media/magic-metadata"; import { batch } from "ente-utils/array"; @@ -102,13 +102,13 @@ export const getCollectionUserFacingName = (collection: Collection) => { * * @param type The type of the new collection. * - * @param magicMetadata Optional metadata to use as the collection's private + * @param magicMetadataData Optional metadata to use as the collection's private * mutable metadata when creating the new collection. */ export const createCollection2 = async ( name: string, type: CollectionType, - magicMetadata?: CollectionPrivateMagicMetadata, + magicMetadataData?: CollectionPrivateMagicMetadata, ): Promise => { const masterKey = await ensureMasterKeyFromSession(); const collectionKey = await generateKey(); @@ -116,10 +116,12 @@ export const createCollection2 = async ( await encryptBox(collectionKey, masterKey); const { encryptedData: encryptedName, nonce: nameDecryptionNonce } = await encryptBox(new TextEncoder().encode(name), collectionKey); - const remoteMagicMetadata = await encryptMagicMetadata( - createMagicMetadataEnvelope(magicMetadata), - collectionKey, - ); + const magicMetadata = magicMetadataData + ? await encryptMagicMetadata( + createMagicMetadata(magicMetadataData), + collectionKey, + ) + : undefined; const remoteCollection = await postCollections({ encryptedKey, @@ -127,9 +129,8 @@ export const createCollection2 = async ( encryptedName, nameDecryptionNonce, type, - ...(remoteMagicMetadata ? { magicMetadata: remoteMagicMetadata } : {}), + ...(magicMetadata ? { magicMetadata } : {}), }); - return decryptRemoteCollectionUsingKey(remoteCollection, masterKey); }; From 64e671b0d218937fa37770d96d559babde999444 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 14:39:49 +0530 Subject: [PATCH 08/11] Try without it first --- web/packages/media/magic-metadata.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/web/packages/media/magic-metadata.ts b/web/packages/media/magic-metadata.ts index 3f043aac15..57af979a29 100644 --- a/web/packages/media/magic-metadata.ts +++ b/web/packages/media/magic-metadata.ts @@ -202,16 +202,11 @@ export const createMagicMetadata = (data: unknown) => { * * This is meant as the inverse of {@link encryptMagicMetadata}, see that * function's documentation for details. - * - * As a usability convenience, this function allows passing `undefined` as the - * {@link remoteMagicMetadata}. In such cases, it will return `undefined` too. */ export const decryptMagicMetadata = async ( - remoteMagicMetadata: RemoteMagicMetadata | undefined, + remoteMagicMetadata: RemoteMagicMetadata, key: string, ): Promise => { - if (!remoteMagicMetadata) return undefined; - const { version, count, From d535cfc5a4e0c14c7dca9b9d04aa7ff012b9f737 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 15:37:59 +0530 Subject: [PATCH 09/11] sk 3 --- .../photos/src/services/collectionService.ts | 5 ++ web/packages/media/collection.ts | 62 +++++++++++++++---- .../new/photos/services/collection.ts | 5 +- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index e0e568434f..82f7fc7521 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -1,6 +1,7 @@ import type { User } from "ente-accounts/services/user"; import { ensureLocalUser } from "ente-accounts/services/user"; import { encryptMetadataJSON, sharedCryptoWorker } from "ente-base/crypto"; +import { isDevBuild } from "ente-base/env"; import log from "ente-base/log"; import { apiURL } from "ente-base/origins"; import { ensureMasterKeyFromSession } from "ente-base/session"; @@ -23,6 +24,7 @@ import { EncryptedMagicMetadata, EnteFile } from "ente-media/file"; import { ItemVisibility } from "ente-media/file-metadata"; import { addToCollection, + createCollection2, isDefaultHiddenCollection, moveToCollection, } from "ente-new/photos/services/collection"; @@ -57,6 +59,9 @@ const FAVORITE_COLLECTION_NAME = "Favorites"; const REQUEST_BATCH_SIZE = 1000; export const createAlbum = (albumName: string) => { + if (isDevBuild && process.env.NEXT_PUBLIC_ENTE_WIP_NEWIMPL) { + return createCollection2(albumName, "album"); + } return createCollection(albumName, "album"); }; diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 466a666c48..1a0ca9fd19 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -1,3 +1,4 @@ +import { decryptBox, decryptBoxBytes } from "ente-base/crypto"; import { type EncryptedMagicMetadata, type MagicMetadataCore, @@ -5,7 +6,11 @@ import { import { ItemVisibility } from "ente-media/file-metadata"; import { nullToUndefined } from "ente-utils/transform"; import { z } from "zod/v4"; -import { RemoteMagicMetadataSchema } from "./magic-metadata"; +import { + decryptMagicMetadata, + RemoteMagicMetadataSchema, + type RemoteMagicMetadata, +} from "./magic-metadata"; // TODO: Audit this file @@ -140,8 +145,8 @@ export interface CollectionUser { */ export const RemoteCollectionUser = z.object({ id: z.number(), - email: z.string().nullish(), - role: z.string().nullish(), + email: z.string().nullish().transform(nullToUndefined), + role: z.string().nullish().transform(nullToUndefined), }); type RemoteCollectionUser = z.infer; @@ -429,22 +434,55 @@ export const decryptRemoteCollection = ( * Decrypt a remote collection using the provided key. * * This is the lower level implementation of {@link decryptRemoteCollection} for - * use by code that already has determined which key should be used for + * use by code that has already determined which key should be used for * decrypting the collection. * - * @param remoteCollection The collection to decrypt. + * @param collection The collection to decrypt. * - * @param key The base64 encoded key to use for decrypting the various encrypted - * fields in {@link remoteCollection}. + * @param decryptionKey The base64 encoded key to use for decrypting the various encrypted + * fields in {@link collection}. * * @returns A decrypted collection ({@link Collection2}). */ -export const decryptRemoteCollectionUsingKey = ( - remoteCollection: RemoteCollection, - key: string, +export const decryptRemoteCollectionUsingKey = async ( + collection: RemoteCollection, + decryptionKey: string, ): Promise => { - // eslint-disable-next-line @typescript-eslint/no-base-to-string - throw new Error("TODO" + remoteCollection.toString() + key.toString()); + const { id, owner, type, sharees, publicURLs, updationTime } = collection; + const { encryptedKey, keyDecryptionNonce } = collection; + const collectionKey = await decryptBox( + { encryptedData: encryptedKey, nonce: keyDecryptionNonce }, + decryptionKey, + ); + + const name = + collection.name ?? + new TextDecoder().decode( + await decryptBoxBytes( + { + encryptedData: collection.encryptedName!, + nonce: collection.nameDecryptionNonce!, + }, + collectionKey, + ), + ); + + const decryptMM = async (mm: RemoteMagicMetadata | undefined) => + mm ? await decryptMagicMetadata(mm, collectionKey) : undefined; + + return { + id, + owner, + key: collectionKey, + name, + type, + sharees, + publicURLs, + updationTime, + magicMetadata: await decryptMM(collection.magicMetadata), + pubMagicMetadata: await decryptMM(collection.pubMagicMetadata), + sharedMagicMetadata: await decryptMM(collection.sharedMagicMetadata), + }; }; /** diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index 2755369673..638d7750a6 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -10,7 +10,7 @@ import { type Collection, type Collection2, type CollectionNewParticipantRole, - type CollectionPrivateMagicMetadata, + type CollectionPrivateMagicMetadataData, type CollectionType, type RemoteCollection, } from "ente-media/collection"; @@ -108,7 +108,7 @@ export const getCollectionUserFacingName = (collection: Collection) => { export const createCollection2 = async ( name: string, type: CollectionType, - magicMetadataData?: CollectionPrivateMagicMetadata, + magicMetadataData?: CollectionPrivateMagicMetadataData, ): Promise => { const masterKey = await ensureMasterKeyFromSession(); const collectionKey = await generateKey(); @@ -131,6 +131,7 @@ export const createCollection2 = async ( type, ...(magicMetadata ? { magicMetadata } : {}), }); + return decryptRemoteCollectionUsingKey(remoteCollection, masterKey); }; From 7e13f8b1cc7021a73c9ae2736c85433f263c2cb1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 16:32:58 +0530 Subject: [PATCH 10/11] Rearrange --- web/packages/media/collection.ts | 71 +++---------------- .../new/photos/services/collection.ts | 29 +++++++- 2 files changed, 37 insertions(+), 63 deletions(-) diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 1a0ca9fd19..1ec4b451d8 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -1,4 +1,4 @@ -import { decryptBox, decryptBoxBytes } from "ente-base/crypto"; +import { decryptBoxBytes } from "ente-base/crypto"; import { type EncryptedMagicMetadata, type MagicMetadataCore, @@ -391,69 +391,20 @@ export interface PublicURL { } /** - * Pertinent information about the Ente user on whose behalf we are trying to - * decrypt a collection received from remote. + * Decrypt a remote collection using the provided {@link collectionKey}. + * + * @param collection The remote collection to decrypt. + * + * @param collectionKey The base64 encoded key to use for decrypting the various + * encrypted fields in {@link collection}. + * + * @returns A decrypted collection. */ -interface CollectionDecryptionUser { - /** - * The ID of the currently logged in user. - */ - userID: number; - /** - * The base64 encoded master key of the currently logged in user. - * - * This is used for decrypting collections that the user owns. - */ - masterKey: string; - /** - * The base64 encoded private key of the currently logged in user. - * - * This is used for decrypting collections that have been shared with the - * user. - */ - privateKey: string; -} - -/** - * Decrypt a collection obtained from remote ({@link RemoteCollection}) into the - * collection object that we use and persist on the client ({@link Collection}). - * - * @param remoteCollection The collection object we received from remote. This - * can be thought of as an envelope, the actual consumables within it are - * encrypted using the user's master key or public key. - * - * @param user The ID of the currently logged in user, and their keys. This is - * needed to decrypt the encrypted remote collection. - */ -export const decryptRemoteCollection = ( - remoteCollection: RemoteCollection, - user: CollectionDecryptionUser, -): Promise => {}; - -/** - * Decrypt a remote collection using the provided key. - * - * This is the lower level implementation of {@link decryptRemoteCollection} for - * use by code that has already determined which key should be used for - * decrypting the collection. - * - * @param collection The collection to decrypt. - * - * @param decryptionKey The base64 encoded key to use for decrypting the various encrypted - * fields in {@link collection}. - * - * @returns A decrypted collection ({@link Collection2}). - */ -export const decryptRemoteCollectionUsingKey = async ( +export const decryptRemoteCollection = async ( collection: RemoteCollection, - decryptionKey: string, + collectionKey: string, ): Promise => { const { id, owner, type, sharees, publicURLs, updationTime } = collection; - const { encryptedKey, keyDecryptionNonce } = collection; - const collectionKey = await decryptBox( - { encryptedData: encryptedKey, nonce: keyDecryptionNonce }, - decryptionKey, - ); const name = collection.name ?? diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index 638d7750a6..5a27c94458 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -1,11 +1,11 @@ import type { User } from "ente-accounts/services/user"; -import { boxSeal, encryptBox, generateKey } from "ente-base/crypto"; +import { boxSeal, decryptBox, encryptBox, generateKey } from "ente-base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; import { apiURL } from "ente-base/origins"; import { ensureMasterKeyFromSession } from "ente-base/session"; import { CollectionSubType, - decryptRemoteCollectionUsingKey, + decryptRemoteCollection, RemoteCollectionSchema, type Collection, type Collection2, @@ -132,7 +132,30 @@ export const createCollection2 = async ( ...(magicMetadata ? { magicMetadata } : {}), }); - return decryptRemoteCollectionUsingKey(remoteCollection, masterKey); + return decryptRemoteCollectionUsingMasterKey(remoteCollection); +}; + +/** + * Decrypt a collection obtained from remote ({@link RemoteCollection}) into the + * collection object that we use and persist on the client ({@link Collection}). + * + * @param collection The collection object we received from remote. This + * can be thought of as an envelope, the actual consumables within it are + * encrypted using the user's master key or public key. + * + * @param user The ID of the currently logged in user, and their keys. This is + * needed to decrypt the encrypted remote collection. + */ +export const decryptRemoteCollectionUsingMasterKey = async ( + collection: RemoteCollection, +): Promise => { + const { encryptedKey, keyDecryptionNonce } = collection; + const collectionKey = await decryptBox( + { encryptedData: encryptedKey, nonce: keyDecryptionNonce }, + await ensureMasterKeyFromSession(), + ); + + return decryptRemoteCollection(collection, collectionKey); }; /** From b5009126507308801dd077a8a49c865314543e0b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 12 Jun 2025 16:59:44 +0530 Subject: [PATCH 11/11] Refactor --- .../photos/src/services/collectionService.ts | 4 +- web/packages/media/collection.ts | 65 +++++-------------- .../new/photos/services/collection.ts | 54 ++++++++------- web/packages/utils/transform.ts | 7 -- 4 files changed, 50 insertions(+), 80 deletions(-) diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index 82f7fc7521..a2ef44a624 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -60,7 +60,9 @@ const REQUEST_BATCH_SIZE = 1000; export const createAlbum = (albumName: string) => { if (isDevBuild && process.env.NEXT_PUBLIC_ENTE_WIP_NEWIMPL) { - return createCollection2(albumName, "album"); + // TODO: WIP + console.log(createCollection2); + // return createCollection2(albumName, "album"); } return createCollection(albumName, "album"); }; diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts index 1ec4b451d8..bf0f84fdab 100644 --- a/web/packages/media/collection.ts +++ b/web/packages/media/collection.ts @@ -4,7 +4,7 @@ import { type MagicMetadataCore, } from "ente-media/file"; import { ItemVisibility } from "ente-media/file-metadata"; -import { nullToUndefined } from "ente-utils/transform"; +import { nullishToEmpty, nullToUndefined } from "ente-utils/transform"; import { z } from "zod/v4"; import { decryptMagicMetadata, @@ -12,8 +12,6 @@ import { type RemoteMagicMetadata, } from "./magic-metadata"; -// TODO: Audit this file - /** * The type of a collection. * @@ -155,47 +153,22 @@ type RemoteCollectionUser = z.infer; * Zod schema for {@link Collection}. * * See: [Note: Schema suffix for exported Zod schemas]. - * - * TODO(RE): The following reasoning is not fully correct, since we anyways need - * to do a conversion when decrypting the collection's fields. The intent is to - * update this once we've figured out something that also works with the current - * persistence mechanism (trying to avoid a migration). - * - * [Note: Schema validation for bulk persisted remote objects] - * - * Objects like files and collections that we get from remote are treated - * specially when it comes to schema validation. - * - * 4. Casting instead of validating when reading local values. - * - * Let us take a concrete example of the {@link Collection} TypeScript type, - * whose zod schema is defined by {@link RemoteCollection}. - * - * While we always use the {@link RemoteCollection} schema validator when - * parsing remote network responses, we don't do the same when reading the - * persisted values. This is to retain the performance characteristics of the - * existing code. This might seem miniscule for the {@link Collection} example, - * but users can easily have hundreds of thousands of {@link EnteFile}s - * persisted locally, and while the overhead of validation when reading from DB - * might not matter, but it needs to be profiled first before adding it to the - * existing code paths. - * - * So when reading arrays of these objects from local DB, we do a cast instead - * of do a runtime zod validation. - * - * This means that the "types might lie". On the other hand, we need to special - * case only very specific objects this way: - * - * 1. {@link EnteFile} - * 2. {@link Collection} - * 3. {@link Trash} - * */ export const RemoteCollectionSchema = z.object({ id: z.number(), owner: RemoteCollectionUser, encryptedKey: z.string(), + /** + * Remote will set this to a blank string for albums which have been shared + * with the user (the decryption pipeline for those doesn't use the nonce). + */ keyDecryptionNonce: z.string(), + /** + * Expected to be present (along with {@link nameDecryptionNonce}), but it + * is still optional since it might not be present if {@link name} is present. + */ + encryptedName: z.string().nullish().transform(nullToUndefined), + nameDecryptionNonce: z.string().nullish().transform(nullToUndefined), /** * Not used anymore, but still might be present for very old collections. * @@ -206,17 +179,11 @@ export const RemoteCollectionSchema = z.object({ */ name: z.string().nullish().transform(nullToUndefined), /** - * Expected to be present (along with {@link nameDecryptionNonce}), but it - * is still optional since it might not be present if {@link name} is present. + * Expected to be one of {@link CollectionType} */ - encryptedName: z.string().nullish().transform(nullToUndefined), - nameDecryptionNonce: z.string().nullish().transform(nullToUndefined), - /* Expected to be one of {@link CollectionType} */ type: z.string(), - // TODO(RE): Use nullishToEmpty? - sharees: z.array(RemoteCollectionUser).nullish().transform(nullToUndefined), // ? - // TODO(RE): Use nullishToEmpty? - publicURLs: z.array(z.looseObject({})).nullish().transform(nullToUndefined), // ? + sharees: z.array(RemoteCollectionUser).nullish().transform(nullishToEmpty), + publicURLs: z.array(z.looseObject({})).nullish().transform(nullishToEmpty), updationTime: z.number(), /** * Tombstone marker. @@ -326,7 +293,7 @@ export interface Collection2 { * * Expected to be one of {@link CollectionType}. */ - type: string; // CollectionType; + type: string; /** * The other Ente users with whom the collection has been shared with. * @@ -335,7 +302,7 @@ export interface Collection2 { * - {@link email} will be set. * - {@link role} is expected to be one of "VIEWER" or "COLLABORATOR". */ - sharees?: CollectionUser[]; + sharees: CollectionUser[]; /** * Public links that can be used to access and update the collection. */ diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts index 5a27c94458..11633e2010 100644 --- a/web/packages/new/photos/services/collection.ts +++ b/web/packages/new/photos/services/collection.ts @@ -1,5 +1,11 @@ -import type { User } from "ente-accounts/services/user"; -import { boxSeal, decryptBox, encryptBox, generateKey } from "ente-base/crypto"; +import { ensureLocalUser, type User } from "ente-accounts/services/user"; +import { + boxSeal, + boxSealOpen, + decryptBox, + encryptBox, + generateKey, +} from "ente-base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "ente-base/http"; import { apiURL } from "ente-base/origins"; import { ensureMasterKeyFromSession } from "ente-base/session"; @@ -22,7 +28,7 @@ import { } from "ente-media/magic-metadata"; import { batch } from "ente-utils/array"; import { z } from "zod/v4"; -import { getPublicKey } from "./user"; +import { ensureUserKeyPair, getPublicKey } from "./user"; /** * An reasonable but otherwise arbitrary number of items (e.g. files) to include @@ -123,7 +129,7 @@ export const createCollection2 = async ( ) : undefined; - const remoteCollection = await postCollections({ + const collection = await postCollections({ encryptedKey, keyDecryptionNonce, encryptedName, @@ -132,30 +138,32 @@ export const createCollection2 = async ( ...(magicMetadata ? { magicMetadata } : {}), }); - return decryptRemoteCollectionUsingMasterKey(remoteCollection); + return decryptRemoteCollection( + collection, + await decryptCollectionKey(collection), + ); }; /** - * Decrypt a collection obtained from remote ({@link RemoteCollection}) into the - * collection object that we use and persist on the client ({@link Collection}). - * - * @param collection The collection object we received from remote. This - * can be thought of as an envelope, the actual consumables within it are - * encrypted using the user's master key or public key. - * - * @param user The ID of the currently logged in user, and their keys. This is - * needed to decrypt the encrypted remote collection. + * Return the decrypted collection key (as a base64 string) for the given + * {@link RemoteCollection}. */ -export const decryptRemoteCollectionUsingMasterKey = async ( +export const decryptCollectionKey = async ( collection: RemoteCollection, -): Promise => { - const { encryptedKey, keyDecryptionNonce } = collection; - const collectionKey = await decryptBox( - { encryptedData: encryptedKey, nonce: keyDecryptionNonce }, - await ensureMasterKeyFromSession(), - ); - - return decryptRemoteCollection(collection, collectionKey); +): Promise => { + const { owner, encryptedKey, keyDecryptionNonce } = collection; + if (owner.id == ensureLocalUser().id) { + // The collection key of collections owned by the user is encrypted with + // the user's master key. + return decryptBox( + { encryptedData: encryptedKey, nonce: keyDecryptionNonce }, + await ensureMasterKeyFromSession(), + ); + } else { + // The collection key of collections shared with the user is encrypted + // with the user's public key. + return boxSealOpen(encryptedKey, await ensureUserKeyPair()); + } }; /** diff --git a/web/packages/utils/transform.ts b/web/packages/utils/transform.ts index 11c16b5472..905977c6d7 100644 --- a/web/packages/utils/transform.ts +++ b/web/packages/utils/transform.ts @@ -4,13 +4,6 @@ export const nullToUndefined = (v: T | null | undefined): T | undefined => v === null ? undefined : v; -/** - * Convert any falsey value (including blank strings) to `undefined`; - * passthrough everything else unchanged. - */ -export const falseyToUndefined = (v: T | null | undefined): T | undefined => - v || undefined; - /** * Convert `null` and `undefined` to `0`, passthrough everything else unchanged. */