TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] File internals cleanup (#6320)

Browse Source
This commit is contained in:
Manav Rathi
2025-06-20 18:03:04 +05:30
committed by GitHub
parent 57d919b56f 63fc066301
commit 98fd2021b7
17 changed files with 142 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
web/apps/photos/src/components/FixCreationTime.tsx
View File

@@ -18,7 +18,6 @@ import { downloadManager } from "ente-gallery/services/download";
import { extractExifDates } from "ente-gallery/services/exif";
import { fileLogID, type EnteFile } from "ente-media/file";
import {
decryptPublicMagicMetadata,
fileCreationPhotoDate,
fileFileName,
type ParsedMetadataDate,
@@ -343,10 +342,7 @@ const updateFileDate = async (
if (!newDate) return;
const existingDate = fileCreationPhotoDate(
file,
await decryptPublicMagicMetadata(file),
);
const existingDate = fileCreationPhotoDate(file);
if (newDate.timestamp == existingDate.getTime()) return;
await updateFilePublicMagicMetadata(file, {

28
web/apps/photos/src/pages/gallery.tsx
View File

@@ -38,10 +38,7 @@ import { FullScreenDropZone } from "ente-gallery/components/FullScreenDropZone";
import { type UploadTypeSelectorIntent } from "ente-gallery/components/Upload";
import { type Collection } from "ente-media/collection";
import { type EnteFile } from "ente-media/file";
import {
updateRemotePrivateMagicMetadata,
type ItemVisibility,
} from "ente-media/file-metadata";
import { type ItemVisibility } from "ente-media/file-metadata";
import {
CollectionSelector,
type CollectionSelectorAttributes,
@@ -78,6 +75,7 @@ import {
} from "ente-new/photos/services/collection-summary";
import { getAllLocalCollections } from "ente-new/photos/services/collections";
import exportService from "ente-new/photos/services/export";
import { updateFilesVisibility } from "ente-new/photos/services/file";
import {
getLocalFiles,
getLocalTrashedFiles,
@@ -847,14 +845,26 @@ const Page: React.FC = () => {
const fileID = file.id;
dispatch({ type: "addPendingVisibilityUpdate", fileID });
try {
const privateMagicMetadata =
await updateRemotePrivateMagicMetadata(file, {
visibility,
});
await updateFilesVisibility([file], visibility);
// [Note: Interactive updates to file metadata]
//
// 1. Update the remote metadata.
//
// 2. Construct a fake a metadata object with the updates
// reflected in it.
//
// 3. The caller (eventually) triggers a remote sync in the
// background, but meanwhile uses this updated metadata.
//
// TODO(RE): Replace with file fetch?
dispatch({
type: "unsyncedPrivateMagicMetadataUpdate",
fileID,
privateMagicMetadata,
privateMagicMetadata: {
...file.magicMetadata,
version: (file.magicMetadata?.version ?? 0) + 1,
data: { ...file.magicMetadata?.data, visibility },
},
});
} finally {
dispatch({ type: "removePendingVisibilityUpdate", fileID });

11
web/packages/gallery/components/FileInfo.tsx
View File

@@ -58,11 +58,9 @@ import { tagNumericValue, type RawExifTags } from "ente-gallery/services/exif";
import { formattedByteSize } from "ente-gallery/utils/units";
import { type EnteFile } from "ente-media/file";
import {
fileCaption,
fileCreationPhotoDate,
fileFileName,
fileLocation,
filePublicMagicMetadata,
type ParsedMetadata,
type ParsedMetadataDate,
} from "ente-media/file-metadata";
@@ -260,7 +258,7 @@ export const FileInfo: React.FC<FileInfoProps> = ({
onSelectPerson?.(personID);
};
const uploaderName = filePublicMagicMetadata(file)?.uploaderName;
const uploaderName = file.pubMagicMetadata?.data.uploaderName;
return (
<FileInfoSidebar {...{ open, onClose }}>
@@ -582,7 +580,7 @@ const Caption: React.FC<CaptionProps> = ({
}) => {
const [isSaving, setIsSaving] = useState(false);
const caption = fileCaption(file) ?? "";
const caption = file.pubMagicMetadata?.data.caption ?? "";
const formik = useFormik<{ caption: string }>({
initialValues: { caption },
@@ -675,10 +673,7 @@ const CreationTime: React.FC<CreationTimeProps> = ({
const [isEditing, setIsEditing] = useState(false);
const [isSaving, setIsSaving] = useState(false);
const originalDate = fileCreationPhotoDate(
file,
filePublicMagicMetadata(file),
);
const originalDate = fileCreationPhotoDate(file);
const saveEdits = async (pickedTime: ParsedMetadataDate) => {
setIsEditing(false);

8
web/packages/gallery/components/viewer/FileViewer.tsx
View File

@@ -37,11 +37,7 @@ import {
type FileInfoProps,
} from "ente-gallery/components/FileInfo";
import type { Collection } from "ente-media/collection";
import {
fileFileName,
fileVisibility,
ItemVisibility,
} from "ente-media/file-metadata";
import { fileFileName, ItemVisibility } from "ente-media/file-metadata";
import { FileType } from "ente-media/file-type";
import type { EnteFile } from "ente-media/file.js";
import { isHEICExtension, needsJPEGConversion } from "ente-media/formats";
@@ -670,7 +666,7 @@ export const FileViewer: React.FC<FileViewerProps> = ({
file &&
activeAnnotatedFile.annotation.showArchive
) {
switch (fileVisibility(file)) {
switch (file.magicMetadata?.data.visibility) {
case undefined:
case ItemVisibility.visible:
isArchived = false;

6
web/packages/gallery/components/viewer/data-source.ts
View File

@@ -7,7 +7,6 @@ import {
type HLSPlaylistDataForFile,
} from "ente-gallery/services/video";
import type { EnteFile } from "ente-media/file";
import { fileCaption, filePublicMagicMetadata } from "ente-media/file-metadata";
import { FileType } from "ente-media/file-type";
import { ensureString } from "ente-utils/ensure";
@@ -439,7 +438,7 @@ const enqueueUpdates = async (
const update = (itemData: Partial<ItemData>, validTill?: Date) => {
// Use the file's caption as its alt text (in addition to using it as
// the visible caption).
const alt = fileCaption(file);
const alt = file.pubMagicMetadata?.data.caption;
_state.itemDataByFileID.set(file.id, {
...itemData,
@@ -647,8 +646,7 @@ const thumbnailDimensions = (
{ width: thumbnailWidth, height: thumbnailHeight }: Partial<ItemData>,
file: EnteFile,
) => {
const { w: imageWidth, h: imageHeight } =
filePublicMagicMetadata(file) ?? {};
const { w: imageWidth, h: imageHeight } = file.pubMagicMetadata?.data ?? {};
if (thumbnailWidth && thumbnailHeight && imageWidth && imageHeight) {
const arThumb = thumbnailWidth / thumbnailHeight;
const arImage = imageWidth / imageHeight;

122
web/packages/gallery/services/file.ts
View File

@@ -1,122 +0,0 @@
/* TODO: Audit this file */
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { encryptMetadataJSON } from "ente-base/crypto";
import { apiURL } from "ente-base/origins";
import { updateMagicMetadata } from "ente-gallery/services/magic-metadata";
import type {
EncryptedMagicMetadata,
EnteFile,
FilePublicMagicMetadata,
FilePublicMagicMetadataProps,
FileWithUpdatedPublicMagicMetadata,
} from "ente-media/file";
import { mergeMetadata } from "ente-media/file";
import HTTPService from "ente-shared/network/HTTPService";
import { getToken } from "ente-shared/storage/localStorage/helpers";
export interface UpdateMagicMetadataRequest {
id: number;
magicMetadata: EncryptedMagicMetadata;
}
interface BulkUpdateMagicMetadataRequest {
metadataList: UpdateMagicMetadataRequest[];
}
export const updateFilePublicMagicMetadata = async (
fileWithUpdatedPublicMagicMetadataList: FileWithUpdatedPublicMagicMetadata[],
): Promise<EnteFile[]> => {
const token = getToken();
if (!token) {
// @ts-ignore
return;
}
const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] };
for (const {
file,
updatedPublicMagicMetadata,
} of fileWithUpdatedPublicMagicMetadataList) {
const { encryptedData, decryptionHeader } = await encryptMetadataJSON(
updatedPublicMagicMetadata.data,
file.key,
);
reqBody.metadataList.push({
id: file.id,
magicMetadata: {
version: updatedPublicMagicMetadata.version,
count: updatedPublicMagicMetadata.count,
data: encryptedData,
header: decryptionHeader,
},
});
}
await HTTPService.put(
await apiURL("/files/public-magic-metadata"),
reqBody,
// @ts-ignore
null,
{ "X-Auth-Token": token },
);
return fileWithUpdatedPublicMagicMetadataList.map(
({ file, updatedPublicMagicMetadata }): EnteFile => ({
...file,
pubMagicMetadata: {
...updatedPublicMagicMetadata,
version: updatedPublicMagicMetadata.version + 1,
},
}),
);
};
export async function changeFileName(
file: EnteFile,
editedName: string,
): Promise<EnteFile> {
const updatedPublicMagicMetadataProps: FilePublicMagicMetadataProps = {
editedName,
};
const updatedPublicMagicMetadata: FilePublicMagicMetadata =
await updateMagicMetadata(
updatedPublicMagicMetadataProps,
file.pubMagicMetadata,
file.key,
);
const updateResult = await updateFilePublicMagicMetadata([
{ file, updatedPublicMagicMetadata },
]);
// @ts-ignore
return updateResult[0];
}
export async function changeCaption(
file: EnteFile,
caption: string,
): Promise<EnteFile> {
const updatedPublicMagicMetadataProps: FilePublicMagicMetadataProps = {
caption,
};
const updatedPublicMagicMetadata: FilePublicMagicMetadata =
await updateMagicMetadata(
updatedPublicMagicMetadataProps,
file.pubMagicMetadata,
file.key,
);
const updateResult = await updateFilePublicMagicMetadata([
{ file, updatedPublicMagicMetadata },
]);
// @ts-ignore
return updateResult[0];
}
export function updateExistingFilePubMetadata(
existingFile: EnteFile,
updatedFile: EnteFile,
) {
// @ts-ignore
existingFile.pubMagicMetadata = updatedFile.pubMagicMetadata;
// @ts-ignore
existingFile.metadata = mergeMetadata([existingFile])[0].metadata;
}

9
web/packages/gallery/services/upload/upload-service.ts
View File

@@ -33,14 +33,13 @@ import type {
EncryptedMagicMetadata,
EnteFile,
FilePublicMagicMetadata,
FilePublicMagicMetadataProps,
} from "ente-media/file";
import {
fileFileName,
metadataHash,
type FileMetadata,
type FilePublicMagicMetadataData,
type ParsedMetadata,
type PublicMagicMetadata,
} from "ente-media/file-metadata";
import { FileType, type FileTypeInfo } from "ente-media/file-type";
import { encodeLivePhoto } from "ente-media/live-photo";
@@ -1047,7 +1046,7 @@ const readEntireStream = async (stream: ReadableStream) =>
interface ExtractAssetMetadataResult {
metadata: FileMetadata;
publicMagicMetadata: FilePublicMagicMetadataProps;
publicMagicMetadata: FilePublicMagicMetadataData;
}
/**
@@ -1171,7 +1170,7 @@ const extractImageOrVideoMetadata = async (
parsedMetadataJSONMap,
);
const publicMagicMetadata: PublicMagicMetadata = {};
const publicMagicMetadata: FilePublicMagicMetadataData = {};
const modificationTime =
parsedMetadataJSON?.modificationTime ?? lastModifiedMs * 1000;
@@ -1490,7 +1489,7 @@ const augmentWithThumbnail = async (
};
const constructPublicMagicMetadata = async (
publicMagicMetadataProps: FilePublicMagicMetadataProps,
publicMagicMetadataProps: FilePublicMagicMetadataData,
): Promise<FilePublicMagicMetadata> => {
const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps(
publicMagicMetadataProps,

11
web/packages/gallery/services/video.ts
View File

@@ -10,11 +10,8 @@ import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { ensureAuthToken } from "ente-base/token";
import { fileLogID, type EnteFile } from "ente-media/file";
import {
filePublicMagicMetadata,
updateRemotePublicMagicMetadata,
} from "ente-media/file-metadata";
import { FileType } from "ente-media/file-type";
import { updateFilePublicMagicMetadata } from "ente-new/photos/services/file";
import {
getAllLocalFiles,
getLocalTrashFileIDs,
@@ -338,7 +335,7 @@ export const hlsPlaylistDataForFile = async (
): Promise<HLSPlaylistDataForFile> => {
ensurePrecondition(file.metadata.fileType == FileType.video);
if (filePublicMagicMetadata(file)?.sv == 1) {
if (file.pubMagicMetadata?.data.sv == 1) {
return "skip";
}
@@ -903,7 +900,7 @@ const backfillQueue = async (
// Not in trash.
!localTrashFileIDs.has(f.id) &&
// See: [Note: Marking files which do not need video processing]
filePublicMagicMetadata(f)?.sv != 1,
f.pubMagicMetadata?.data.sv != 1,
),
);
@@ -1040,7 +1037,7 @@ const processQueueItem = async ({
if (!res) {
log.info(`Generate HLS for ${fileLogID(file)} | not-required`);
// See: [Note: Marking files which do not need video processing]
await updateRemotePublicMagicMetadata(file, { sv: 1 });
await updateFilePublicMagicMetadata(file, { sv: 1 });
return;
}

398
web/packages/media/file-metadata.ts
View File

@@ -1,20 +1,8 @@
import { decryptMetadataJSON, encryptMetadataJSON } from "ente-base/crypto";
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import { apiURL } from "ente-base/origins";
import { type Location } from "ente-base/types";
import {
fileLogID,
type EnteFile,
type EnteFile2,
type FileMagicMetadata,
type FilePrivateMagicMetadata,
type FilePublicMagicMetadata,
} from "ente-media/file";
import { type EnteFile, type EnteFile2 } from "ente-media/file";
import { nullToUndefined } from "ente-utils/transform";
import { z } from "zod/v4";
import { mergeMetadata1 } from "./file";
import { FileType } from "./file-type";
import type { RemoteMagicMetadata } from "./magic-metadata";
/**
* Information about the file that never changes post upload.
@@ -290,10 +278,8 @@ export type ItemVisibility =
* And never like:
*
* foo: T | undefined
*
* Also see: [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet].
*/
export interface PublicMagicMetadata {
export interface FilePublicMagicMetadataData {
/**
* A ISO 8601 date time string without a timezone, indicating the local time
* where the photo (or video) was taken.
@@ -358,6 +344,20 @@ export interface PublicMagicMetadata {
* (The owner of such files will be the owner of the collection)
*/
uploaderName?: string;
/**
* Edited latitude of the file
*
* If the user edits the location (latitude and longitude) of a file within
* Ente, then the edits will be stored as the {@link lat} and {@link long}
* properties in the file's public magic metadata.
*/
lat?: number;
/**
* Edited longitude of the file.
*
* See {@link long}.
*/
long?: number;
/**
* An arbitrary integer set to indicate that this file should be skipped for
* the purpose of HLS generation.
@@ -390,10 +390,6 @@ export interface PublicMagicMetadata {
/**
* Zod schema for the {@link PublicMagicMetadata} type.
*
* See: [Note: Duplicated Zod schema and TypeScript type]
*
* ---
*
* [Note: Use looseObject for metadata Zod schemas]
*
* It is important to (recursively) use the {@link looseObject} option when
@@ -402,60 +398,20 @@ export interface PublicMagicMetadata {
* might be other, newer, clients out there adding fields that the current
* client might not we aware of, and we don't want to overwrite them.
*/
const PublicMagicMetadata = z.looseObject({
// [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet]
//
// Using `optional` is not accurate here. The key is optional, but the
// value itself is not optional.
//
// Zod doesn't work with `exactOptionalPropertyTypes` yet, but it seems
// to be on the roadmap so we suppress these mismatches.
//
// See:
// https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063
editedTime: z.number().optional(),
export const FilePublicMagicMetadataData = z.looseObject({
dateTime: z.string().nullish().transform(nullToUndefined),
offsetTime: z.string().nullish().transform(nullToUndefined),
editedTime: z.number().nullish().transform(nullToUndefined),
editedName: z.string().nullish().transform(nullToUndefined),
w: z.number().nullish().transform(nullToUndefined),
h: z.number().nullish().transform(nullToUndefined),
caption: z.string().nullish().transform(nullToUndefined),
uploaderName: z.string().nullish().transform(nullToUndefined),
lat: z.number().nullish().transform(nullToUndefined),
long: z.number().nullish().transform(nullToUndefined),
sv: z.number().nullish().transform(nullToUndefined),
});
/**
* Return the private magic metadata for an {@link EnteFile}.
*
* We are not expected to be in a scenario where the file gets to the UI without
* having its private magic metadata decrypted, so this function is a sanity
* check and should be a no-op in usually. It'll throw if it finds its
* assumptions broken. Once the types have been refactored this entire
* check/cast shouldn't be needed, and this should become a trivial accessor.
*/
export const filePrivateMagicMetadata = (file: EnteFile) => {
if (!file.magicMetadata) return undefined;
if (typeof file.magicMetadata.data == "string") {
throw new Error(
`Private magic metadata for ${fileLogID(file)} had not been decrypted even when the file reached the UI layer`,
);
}
return file.magicMetadata.data;
};
/**
* Return the public magic metadata for an {@link EnteFile}.
*
* We are not expected to be in a scenario where the file gets to the UI without
* having its public magic metadata decrypted, so this function is a sanity
* check and should be a no-op in usually. It'll throw if it finds its
* assumptions broken. Once the types have been refactored this entire
* check/cast shouldn't be needed, and this should become a trivial accessor.
*/
export const filePublicMagicMetadata = (file: EnteFile) => {
if (!file.pubMagicMetadata) return undefined;
if (typeof file.pubMagicMetadata.data == "string") {
throw new Error(
`Public magic metadata for ${fileLogID(file)} had not been decrypted even when the file reached the UI layer`,
);
}
// This cast is unavoidable in the current setup. We need to refactor the
// types so that this cast in not needed.
return file.pubMagicMetadata.data as PublicMagicMetadata;
};
/**
* Return the hash of the file by reading it from its metadata.
*
@@ -483,59 +439,11 @@ export const metadataHash = (metadata: FileMetadata) => {
};
/**
* Return the public magic metadata for the given {@link file}.
*
* The file we persist in our local db has the metadata in the encrypted form
* that we get it from remote. We decrypt when we read it, and also hang the
* decrypted version to the in-memory {@link EnteFile} as a cache.
*
* If the file doesn't have any public magic metadata attached to it, return
* `undefined`.
* Return `true` if the {@link ItemVisibility} of the given {@link file} is
* archived.
*/
export const decryptPublicMagicMetadata = async (
file: EnteFile,
): Promise<PublicMagicMetadata | undefined> => {
const envelope = file.pubMagicMetadata;
if (!envelope) return undefined;
// TODO: This function can be optimized to directly return the cached value
// instead of reparsing it using Zod. But that requires us (a) first fix the
// types, and (b) guarantee that we're the only ones putting that parsed
// data there, so that it is in a known good state (currently we exist in
// parallel with other functions that do the similar things).
const jsonValue =
typeof envelope.data == "string"
? await decryptMetadataJSON(
{
encryptedData: envelope.data,
decryptionHeader: envelope.header,
},
file.key,
)
: envelope.data;
const result = PublicMagicMetadata.parse(
// TODO: Can we avoid this cast?
withoutNullAndUndefinedValues(jsonValue as object),
);
// -@ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet]
// We can't use -@ts-expect-error since this code is also included in the
// packages which don't have strict mode enabled (and thus don't error).
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
envelope.data = result;
// -@ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet]
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
return result;
};
const withoutNullAndUndefinedValues = (o: object) =>
Object.fromEntries(
Object.entries(o).filter(([, v]) => v !== null && v !== undefined),
);
export const isArchivedFile = (file: EnteFile) =>
file.magicMetadata?.data.visibility == ItemVisibility.archived;
/**
* Return the file name of the file (including both the name and the extension).
@@ -555,241 +463,16 @@ export const fileFileName = (file: EnteFile | EnteFile2) =>
* Return the file's creation date as a Date in the hypothetical "timezone of
* the photo".
*
* For all the details and nuance, see {@link createPhotoDate}.
* This function handles files with edited dates. For all the details and
* nuance, see {@link createPhotoDate}.
*/
export const fileCreationPhotoDate = (
file: EnteFile,
publicMagicMetadata: PublicMagicMetadata | undefined,
) =>
export const fileCreationPhotoDate = (file: EnteFile) =>
createPhotoDate(
publicMagicMetadata?.dateTime ??
publicMagicMetadata?.editedTime ??
file.pubMagicMetadata?.data.dateTime ??
file.pubMagicMetadata?.data.editedTime ??
file.metadata.creationTime,
);
/**
* Update the private magic metadata associated with a file on remote.
*
* @param file The {@link EnteFile} whose public magic metadata we want to
* update.
*
* @param metadataUpdates A subset of {@link FilePrivateMagicMetadataData} containing
* the fields that we want to add or update.
*
* @returns An updated {@link FilePrivateMagicMetadataData} object containing the
* (decrypted) metadata updates we just made. This is effectively what we would
* get if we to ask the remote for the latest file for this ID, except we don't
* do an actual sync and instead reconstruct it piecemeal.
*
* [Note: Interactive updates to file metadata]
*
* This function updates the magic metadata on remote, and returns a magic
* metadata object with the updated (and decrypted) values, but it does not
* update the state of the file objects in our databases.
*
* The caller needs to ensure that we subsequently sync with remote to fetch the
* updates as part of the diff and update the {@link EnteFile} that is persisted
* in our local db.
*
* This partial update approach is used because a full sync requires multiple
* API calls, which can cause a slow experience for interactive operations (e.g.
* archiving a file). So this function does not immediately perform the sync,
* but instead expects the caller to arrange for an eventual delayed sync in the
* background without waiting for it to complete.
*
* Returning a modified in-memory object is essential because in addition to the
* updated metadata itself, the metadatum (See: [Note: Metadatum]) contain a
* version field that is incremented for each change. So if we were not to
* update the version, and if the user were to perform another operation on that
* file before the asynchronous remote sync completes, the client will send a
* stale version of the metadata, and remote will reject the update.
*
* The overall sequence is thus:
*
* 1. This function modifies the remote metadata.
*
* 2. It returns a metadata object with the updates reflected in it.
*
* 3. The caller (eventually) triggers a remote sync in the background, but
* meanwhile uses this updated metadata.
*/
export const updateRemotePrivateMagicMetadata = async (
file: EnteFile,
metadataUpdates: Partial<FilePrivateMagicMetadataData>,
): Promise<FilePrivateMagicMetadata> => {
const existingMetadata = filePrivateMagicMetadata(file);
const updatedMetadata = { ...(existingMetadata ?? {}), ...metadataUpdates };
const metadataVersion = file.magicMetadata?.version ?? 1;
const updateRequest = await updateMagicMetadataRequest(
file,
updatedMetadata,
metadataVersion,
);
const updatedEnvelope = updateRequest.metadataList[0]!.magicMetadata;
await putFilesPrivateMagicMetadata(updateRequest);
// See: [Note: Interactive updates to file metadata]
// Use the updated envelope we sent as a starting point for the metadata we
// will use for the updated file.
const updatedMagicMetadata = updatedEnvelope as FileMagicMetadata;
// The correct version will come in the updated EnteFile we get in the
// response of the /diff. Temporarily bump it to reflect our latest edit.
updatedMagicMetadata.version = metadataVersion + 1;
// Set the contents (data) to the updated metadata contents we just PUT.
updatedMagicMetadata.data = updatedMetadata;
return updatedMagicMetadata;
};
/**
* Update the public magic metadata associated with a file on remote.
*
* See: [Note: Interactive updates to file metadata]
*
* @param file The {@link EnteFile} whose public magic metadata we want to
* update.
*
* @param metadataUpdates A subset of {@link PublicMagicMetadata} containing the
* fields that we want to add or update.
*/
export const updateRemotePublicMagicMetadata = async (
file: EnteFile,
metadataUpdates: Partial<PublicMagicMetadata>,
) => {
const existingMetadata = await decryptPublicMagicMetadata(file);
const updatedMetadata = { ...(existingMetadata ?? {}), ...metadataUpdates };
const metadataVersion = file.pubMagicMetadata?.version ?? 1;
const updateRequest = await updateMagicMetadataRequest(
file,
updatedMetadata,
metadataVersion,
);
const updatedEnvelope = updateRequest.metadataList[0]!.magicMetadata;
await putFilesPublicMagicMetadata(updateRequest);
// Modify the in-memory object to use the updated envelope. This steps are
// quite ad-hoc, as is the concept of updating the object in place.
file.pubMagicMetadata = updatedEnvelope as FilePublicMagicMetadata;
// The correct version will come in the updated EnteFile we get in the
// response of the /diff. Temporarily bump it for the in place edits.
file.pubMagicMetadata.version = file.pubMagicMetadata.version + 1;
// Re-read the data.
await decryptPublicMagicMetadata(file);
// Re-jig the other bits of EnteFile that depend on its public magic
// metadata.
mergeMetadata1(file);
};
/**
* The shape of the JSON body payload expected by the APIs that update the
* public and private magic metadata fields associated with a file.
*/
interface UpdateMagicMetadataRequest {
/** The list of (file id, new magic metadata) pairs to update */
metadataList: {
/** File ID */
id: number;
/** The new metadata to use */
magicMetadata: RemoteMagicMetadata;
}[];
}
/**
* Construct an remote update request payload from the public or private magic
* metadata JSON object for a {@link file}, using the provided
* {@link encryptMetadataF} function to encrypt the JSON.
*/
const updateMagicMetadataRequest = async (
file: EnteFile,
metadata: FilePrivateMagicMetadataData | PublicMagicMetadata,
metadataVersion: number,
): Promise<UpdateMagicMetadataRequest> => {
// Drop all null or undefined values to obtain the syncable entries.
// See: [Note: Optional magic metadata keys].
const validEntries = Object.entries(metadata).filter(
([, v]) => v !== null && v !== undefined,
);
const { encryptedData, decryptionHeader } = await encryptMetadataJSON(
Object.fromEntries(validEntries),
file.key,
);
return {
metadataList: [
{
id: file.id,
magicMetadata: {
version: metadataVersion,
count: validEntries.length,
data: encryptedData,
header: decryptionHeader,
},
},
],
};
};
/**
* Update the (private) magic metadata for a list of files.
*
* See: [Note: Private magic metadata is called magic metadata on remote]
*
* @param request The list of file ids and the updated encrypted magic metadata
* associated with each of them.
*/
const putFilesPrivateMagicMetadata = async (
request: UpdateMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/files/magic-metadata"), {
method: "PUT",
headers: await authenticatedRequestHeaders(),
body: JSON.stringify(request),
}),
);
/**
* Update the public magic metadata for a list of files.
*
* @param request The list of file ids and the updated encrypted magic metadata
* associated with each of them.
*/
const putFilesPublicMagicMetadata = async (
request: UpdateMagicMetadataRequest,
) =>
ensureOk(
await fetch(await apiURL("/files/public-magic-metadata"), {
method: "PUT",
headers: await authenticatedRequestHeaders(),
body: JSON.stringify(request),
}),
);
/**
* Return the {@link ItemVisibility} for the given {@link file}.
*/
export const fileVisibility = (file: EnteFile) =>
filePrivateMagicMetadata(file)?.visibility;
/**
* Return `true` if the {@link ItemVisibility} of the given {@link file} is
* archived.
*/
export const isArchivedFile = (item: EnteFile) =>
fileVisibility(item) === ItemVisibility.archived;
/**
* Return the GPS coordinates (if any) present in the given {@link EnteFile}.
*/
@@ -848,13 +531,6 @@ export const fileDurationString = (file: EnteFile): string | undefined => {
}
};
/**
* Return the caption, aka "description", (if any) attached to the given
* {@link EnteFile}.
*/
export const fileCaption = (file: EnteFile): string | undefined =>
filePublicMagicMetadata(file)?.caption;
/**
* Metadata about a file extracted from various sources (like Exif) when
* uploading it into Ente.

134
web/packages/media/file.ts
View File

@@ -12,6 +12,7 @@ import {
fileFileName,
FileMetadata,
type FilePrivateMagicMetadataData,
type FilePublicMagicMetadataData,
} from "./file-metadata";
import { FileType } from "./file-type";
import { decryptMagicMetadata, RemoteMagicMetadata } from "./magic-metadata";
@@ -306,7 +307,7 @@ export interface EnteFile
*
* See: [Note: Metadatum]
*/
pubMagicMetadata?: FilePublicMagicMetadata;
pubMagicMetadata?: MagicMetadataCore<FilePublicMagicMetadataData>;
/**
* `true` if this file is in trash (i.e. it has been deleted by the user,
* and will be permanently deleted after 30 days of being moved to trash).
@@ -483,117 +484,12 @@ export const FileDiffResponse = z.object({
hasMore: z.boolean(),
});
export interface FileWithUpdatedPublicMagicMetadata {
file: EnteFile;
updatedPublicMagicMetadata: FilePublicMagicMetadata;
}
export type FileMagicMetadata = MagicMetadataCore<FilePrivateMagicMetadataData>;
export type FilePrivateMagicMetadata =
MagicMetadataCore<FilePrivateMagicMetadataData>;
export interface FilePublicMagicMetadataProps {
/**
* Modified value of the date time associated with an {@link EnteFile}.
*
* Epoch microseconds.
*/
editedTime?: number;
/** See {@link PublicMagicMetadata} in file-metadata.ts */
dateTime?: string;
/** See {@link PublicMagicMetadata} in file-metadata.ts */
offsetTime?: string;
/**
* Edited name of the {@link EnteFile}.
*
* If the user edits the name of the file within Ente, then the edits are
* saved in this field.
*/
editedName?: string;
/**
* A arbitrary textual caption / description that the user has attached to
* the {@link EnteFile}.
*/
caption?: string;
uploaderName?: string;
/**
* Width of the image / video, in pixels.
*/
w?: number;
/**
* Height of the image / video, in pixels.
*/
h?: number;
/**
* Edited latitude for the {@link EnteFile}.
*
* If the user edits the location (latitude and longitude) of a file within
* Ente, then the edits will be stored as the {@link lat} and {@link long}
* properties in the file's public magic metadata.
*/
lat?: number;
/**
* Edited longitude for the {@link EnteFile}.
*
* See {@link long}.
*/
long?: number;
}
export type FilePublicMagicMetadata =
MagicMetadataCore<FilePublicMagicMetadataProps>;
export interface TrashItem extends Omit<EncryptedTrashItem, "file"> {
file: EnteFile;
}
export interface EncryptedTrashItem {
file: EncryptedEnteFile;
/**
* `true` if the file no longer in trash because it was permanently deleted.
*
* This field is relevant when we obtain a trash item as part of the trash
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was permanently deleted.
*/
isDeleted: boolean;
/**
* `true` if the file no longer in trash because it was restored to some
* collection.
*
* This field is relevant when we obtain a trash item as part of the trash
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was restored to a collection.
*/
isRestored: boolean;
deleteBy: number;
createdAt: number;
updatedAt: number;
}
export type Trash = TrashItem[];
/**
* A short identifier for a file in log messages.
*
* e.g. "file flower.png (827233681)"
*
* @returns a string to use as an identifier when logging information about the
* given {@link file}. The returned string contains the file name (for ease of
* debugging) and the file ID (for exactness).
*/
export const fileLogID = (file: EnteFile) =>
`file ${fileFileName(file)} (${file.id})`;
/**
* Return the date when the file will be deleted permanently. Only valid for
* files that are in the user's trash.
*
* This is a convenience wrapper over the {@link deleteBy} property of a file,
* converting that epoch microsecond value into a JavaScript date.
*/
export const enteFileDeletionDate = (file: EnteFile) =>
dateFromEpochMicroseconds(file.deleteBy);
MagicMetadataCore<FilePublicMagicMetadataData>;
export async function decryptFile(
file: EncryptedEnteFile,
@@ -741,7 +637,7 @@ export const decryptRemoteFile = async (
key,
);
// TODO(RE):
const data = genericMM.data as FilePublicMagicMetadataProps;
const data = genericMM.data as FilePublicMagicMetadataData;
// TODO(RE):
pubMagicMetadata = { ...genericMM, header: "", data };
}
@@ -845,6 +741,28 @@ export const mergeMetadata1 = (file: EnteFile): EnteFile => {
export const mergeMetadata = (files: EnteFile[]) =>
files.map((file) => mergeMetadata1(file));
/**
* A short identifier for a file in log messages.
*
* e.g. "file flower.png (827233681)"
*
* @returns a string to use as an identifier when logging information about the
* given {@link file}. The returned string contains the file name (for ease of
* debugging) and the file ID (for exactness).
*/
export const fileLogID = (file: EnteFile) =>
`file ${fileFileName(file)} (${file.id})`;
/**
* Return the date when the file will be deleted permanently. Only valid for
* files that are in the user's trash.
*
* This is a convenience wrapper over the {@link deleteBy} property of a file,
* converting that epoch microsecond value into a JavaScript date.
*/
export const enteFileDeletionDate = (file: EnteFile) =>
dateFromEpochMicroseconds(file.deleteBy);
export interface MagicMetadataCore<T> {
version: number;
count: number;

3
web/packages/new/photos/services/collection.ts
View File

@@ -9,7 +9,6 @@ import {
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import { apiURL } from "ente-base/origins";
import { ensureMasterKeyFromSession } from "ente-base/session";
import type { UpdateMagicMetadataRequest } from "ente-gallery/services/file";
import {
CollectionSubType,
decryptRemoteCollection,
@@ -32,7 +31,7 @@ import {
} from "ente-media/magic-metadata";
import { batch } from "ente-utils/array";
import { z } from "zod/v4";
import { requestBatchSize } from "./file";
import { requestBatchSize, type UpdateMagicMetadataRequest } from "./file";
import { ensureUserKeyPair, getPublicKey } from "./user";
const uncategorizedCollectionName = "Uncategorized";

11
web/packages/new/photos/services/collections.ts
View File

@@ -6,17 +6,16 @@
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { type Collection } from "ente-media/collection";
import {
decryptFile,
type EncryptedTrashItem,
type EnteFile,
type Trash,
} from "ente-media/file";
import { decryptFile, type EnteFile } from "ente-media/file";
import {
getLocalTrash,
getTrashedFiles,
TRASH,
} from "ente-new/photos/services/files";
import {
type EncryptedTrashItem,
type Trash,
} from "ente-new/photos/services/trash";
import HTTPService from "ente-shared/network/HTTPService";
import localForage from "ente-shared/storage/localForage";
import { getToken } from "ente-shared/storage/localStorage/helpers";

7
web/packages/new/photos/services/dedup.ts
View File

@@ -2,10 +2,7 @@ import { ensureLocalUser } from "ente-accounts/services/user";
import { assertionFailed } from "ente-base/assert";
import { newID } from "ente-base/id";
import type { EnteFile } from "ente-media/file";
import {
filePublicMagicMetadata,
metadataHash,
} from "ente-media/file-metadata";
import { metadataHash } from "ente-media/file-metadata";
import {
addToCollection,
createCollectionNameByID,
@@ -312,7 +309,7 @@ const duplicateGroupItemToRetain = (duplicateGroup: DuplicateGroup) => {
const itemsWithCaption: DuplicateGroup["items"] = [];
const itemsWithOtherEdits: DuplicateGroup["items"] = [];
for (const item of duplicateGroup.items) {
const pubMM = filePublicMagicMetadata(item.file);
const pubMM = item.file.pubMagicMetadata?.data;
if (!pubMM) continue;
if (pubMM.caption) itemsWithCaption.push(item);
if (pubMM.editedName ?? pubMM.editedTime)

6
web/packages/new/photos/services/file.ts
View File

@@ -3,8 +3,8 @@ import { apiURL } from "ente-base/origins";
import type { EnteFile, EnteFile2 } from "ente-media/file";
import type {
FilePrivateMagicMetadataData,
FilePublicMagicMetadataData,
ItemVisibility,
PublicMagicMetadata,
} from "ente-media/file-metadata";
import {
createMagicMetadata,
@@ -201,7 +201,7 @@ export const updateFileCaption = (file: EnteFile2, caption: string) =>
*/
export const updateFilePublicMagicMetadata = async (
file: EnteFile2,
updates: PublicMagicMetadata,
updates: FilePublicMagicMetadataData,
) => updateFilesPublicMagicMetadata([file], updates);
/**
@@ -214,7 +214,7 @@ export const updateFilePublicMagicMetadata = async (
*/
const updateFilesPublicMagicMetadata = async (
files: EnteFile2[],
updates: PublicMagicMetadata,
updates: FilePublicMagicMetadataData,
) =>
putFilesPublicMagicMetadata({
metadataList: await Promise.all(

2
web/packages/new/photos/services/files.ts
View File

@@ -7,9 +7,9 @@ import {
mergeMetadata,
type EncryptedEnteFile,
type EnteFile,
type Trash,
} from "ente-media/file";
import { metadataHash } from "ente-media/file-metadata";
import { type Trash } from "ente-new/photos/services/trash";
import HTTPService from "ente-shared/network/HTTPService";
import localForage from "ente-shared/storage/localForage";
import { getToken } from "ente-shared/storage/localStorage/helpers";

3
web/packages/new/photos/services/search/worker.ts
View File

@@ -10,7 +10,6 @@ import {
fileCreationPhotoDate,
fileFileName,
fileLocation,
filePublicMagicMetadata,
} from "ente-media/file-metadata";
import { nullToUndefined } from "ente-utils/transform";
import { z } from "zod/v4";
@@ -385,7 +384,7 @@ const isMatchingFile = (file: EnteFile, suggestion: SearchSuggestion) => {
case "date":
return isDateComponentsMatch(
suggestion.dateComponents,
fileCreationPhotoDate(file, filePublicMagicMetadata(file)),
fileCreationPhotoDate(file),
);
case "location": {

31
web/packages/new/photos/services/trash.ts Normal file
View File

@@ -0,0 +1,31 @@
import type { EncryptedEnteFile, EnteFile } from "ente-media/file";
export interface TrashItem extends Omit<EncryptedTrashItem, "file"> {
file: EnteFile;
}
export interface EncryptedTrashItem {
file: EncryptedEnteFile;
/**
* `true` if the file no longer in trash because it was permanently deleted.
*
* This field is relevant when we obtain a trash item as part of the trash
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was permanently deleted.
*/
isDeleted: boolean;
/**
* `true` if the file no longer in trash because it was restored to some
* collection.
*
* This field is relevant when we obtain a trash item as part of the trash
* diff. It indicates that the file which was previously in trash is no
* longer in the trash because it was restored to a collection.
*/
isRestored: boolean;
deleteBy: number;
createdAt: number;
updatedAt: number;
}
export type Trash = TrashItem[];