diff --git a/web/packages/new/photos/services/ml/embedding.ts b/web/packages/new/photos/services/ml/embedding.ts index fd6e21568e..41b414d972 100644 --- a/web/packages/new/photos/services/ml/embedding.ts +++ b/web/packages/new/photos/services/ml/embedding.ts @@ -108,8 +108,6 @@ const RemoteEmbedding = z.object({ * the crypto layer. */ decryptionHeader: z.string(), - /** Last time (epoch ms) this embedding was updated. */ - updatedAt: z.number(), }); type RemoteEmbedding = z.infer; @@ -339,6 +337,23 @@ const putEmbeddingString = async ( // MARK: - Combined +/** + * The decrypted payload of a {@link RemoteEmbedding} for the "combined" + * {@link EmbeddingModel}. + * + * [Note: Preserve unknown derived data fields] + * + * There is one entry for each of the embedding types that the current client + * knows about. However, there might be other fields apart from the known ones + * at the top level, and we need to ensure that we preserve them verbatim when + * trying use {@link putDerivedData} with an {@link RemoteDerivedData} obtained + * from remote as the base, with locally indexed additions. + */ +export type RemoteDerivedData = Record & { + face: RemoteFaceIndex; + clip: RemoteCLIPIndex; +}; + /** * Update the combined derived data stored for given {@link enteFile} on remote. * This allows other clients to directly pull the derived data instead of @@ -347,24 +362,14 @@ const putEmbeddingString = async ( * The data on remote will be replaced unconditionally, and it is up to the * client (us) to ensure that we preserve the parts of the pre-existing derived * data (if any) that we did not understand or touch. + * + * See: [Note: Preserve unknown derived data fields]. */ export const putDerivedData = async ( enteFile: EnteFile, - remoteFaceIndex: RemoteFaceIndex, - remoteCLIPIndex: RemoteCLIPIndex, -) => { - const combined = { - face: remoteFaceIndex, - clip: remoteCLIPIndex, - }; - log.debug(() => ["Uploading derived data", combined]); - - return putEmbedding( - enteFile, - "combined", - await gzip(JSON.stringify(combined)), - ); -}; + derivedData: RemoteDerivedData, +) => + putEmbedding(enteFile, "combined", await gzip(JSON.stringify(derivedData))); /** * Compress the given {@link string} using "gzip" and return the resultant diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.ts index 5ee0c554c8..139c27ad25 100644 --- a/web/packages/new/photos/services/ml/face.ts +++ b/web/packages/new/photos/services/ml/face.ts @@ -55,6 +55,18 @@ export const faceIndexingVersion = 1; * and {@link RemoteFaceIndex} types respectively. */ export interface FaceIndex { + /** + * The width (in px) of the image (file). + * + * Having the image dimensions here is useful since the coordinates inside + * the {@link Face}s are all normalized (0 to 1) to the width and height of + * the image. + */ + width: number; + /** + * The height (in px) of the image (file). + */ + height: number; /** * The list of faces (and their embeddings) detected in the file. * @@ -211,6 +223,8 @@ export const indexFaces = async ( { data: imageData }: ImageBitmapAndData, electron: MLWorkerElectron, ): Promise => ({ + width: imageData.width, + height: imageData.height, faces: await indexFaces_(enteFile.id, imageData, electron), }); diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 14e6daee35..c3126fc367 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -25,7 +25,7 @@ import { saveFaceIndex, updateAssumingLocalFiles, } from "./db"; -import { pullFaceEmbeddings, putDerivedData } from "./embedding"; +import { putDerivedData } from "./embedding"; import { faceIndexingVersion, indexFaces, type FaceIndex } from "./face"; import type { MLWorkerDelegate, MLWorkerElectron } from "./worker-types"; @@ -208,32 +208,6 @@ export class MLWorker { expose(MLWorker); -/** - * Pull embeddings from remote. - * - * Return true atleast one embedding was pulled. - */ -const pull = async () => { - const res = await Promise.allSettled([ - pullFaceEmbeddings(), - // TODO-ML: clip-test - // pullCLIPEmbeddings(), - ]); - for (const r of res) { - switch (r.status) { - case "fulfilled": - // Return true if any pulled something. - if (r.value) return true; - break; - case "rejected": - // Throw if any failed. - throw r.reason; - } - } - // Return false if neither pulled anything. - return false; -}; - /** * Find out files which need to be indexed. Then index the next batch of them. * @@ -416,20 +390,23 @@ const index = async ( return `Indexed ${nf} faces and clip in ${f} (${ms} ms)`; }); - const remoteFaceIndex = { - ...faceIndex, - version: faceIndexingVersion, - client: userAgent, + const derivedData = { + face: { + version: faceIndexingVersion, + client: userAgent, + ...faceIndex, + }, + clip: { + version: clipIndexingVersion, + client: userAgent, + ...clipIndex, + }, }; - const remoteCLIPIndex = { - ...clipIndex, - version: clipIndexingVersion, - client: userAgent, - }; + log.debug(() => ["Uploading derived data", derivedData]); try { - await putDerivedData(enteFile, remoteFaceIndex, remoteCLIPIndex); + await putDerivedData(enteFile, derivedData); } catch (e) { // See: [Note: Transient and permanent indexing failures] log.error(`Failed to put face index for ${f}`, e);