From 6515d1e7501e739fb1045df7d0d33d0283412584 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 16 Jul 2024 12:12:41 +0530 Subject: [PATCH] WIP refactor data structure --- web/packages/new/photos/services/ml/clip.ts | 2 +- web/packages/new/photos/services/ml/crop.ts | 10 +-- web/packages/new/photos/services/ml/face.ts | 79 +++++++++++---------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index fab1147c37..8fdf1a3d3f 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -14,7 +14,7 @@ export const clipIndexingVersion = 1; /** * The CLIP embedding for a file (and some metadata). * - * See {@link FaceIndex} for a similar structure with more comprehensive + * See {@link RemoteFaceIndex} for a similar structure with more comprehensive * documentation. * * --- diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.ts index ed13cdff52..1810f783df 100644 --- a/web/packages/new/photos/services/ml/crop.ts +++ b/web/packages/new/photos/services/ml/crop.ts @@ -2,7 +2,7 @@ import { blobCache } from "@/next/blob-cache"; import { ensure } from "@/utils/ensure"; import type { EnteFile } from "../../types/file"; import { renderableEnteFileBlob } from "./blob"; -import { type Box, type FaceIndex } from "./face"; +import { type Box, type LocalFaceIndex } from "./face"; import { clamp } from "./math"; /** @@ -16,15 +16,15 @@ import { clamp } from "./math"; * * @param enteFile The {@link EnteFile} whose face crops we want to generate. * - * @param faceIndex The {@link FaceIndex} containing information about the faces - * detected in the given image. + * @param faces The {@link LocalFaceIndex} containing information about the + * faces detected in the given image. * * The generated face crops are saved in a local cache and can subsequently be * retrieved from the {@link BlobCache} named "face-crops". */ export const regenerateFaceCrops = async ( enteFile: EnteFile, - faceIndex: FaceIndex, + faces: Face[], ) => { const imageBitmap = await createImageBitmap( await renderableEnteFileBlob(enteFile), @@ -51,7 +51,7 @@ export const regenerateFaceCrops = async ( */ export const saveFaceCrops = async ( imageBitmap: ImageBitmap, - faceIndex: FaceIndex, + faceIndex: LocalFaceIndex, ) => { const cache = await blobCache("face-crops"); diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.ts index 2fb953e6d7..8e944b3980 100644 --- a/web/packages/new/photos/services/ml/face.ts +++ b/web/packages/new/photos/services/ml/face.ts @@ -36,30 +36,60 @@ export const faceIndexingVersion = 1; /** * The faces in a file (and an embedding for each of them). * - * This interface describes the format of both local and remote face data. + * This interface describes the format of remote face related data (aka "face + * index") for a file. See {@link LocalFaceIndex} for the subset of the fields + * that we also persist locally. * * - Local face detections and embeddings (collectively called as the face - * index) are generated by the current client when uploading a file (or when - * noticing a file which doesn't yet have a face index), stored in the local - * IndexedDB ("ml/db") and also uploaded (E2EE) to remote. + * index) are generated by the current client when uploading a file, or when + * noticing a file which doesn't yet have a face index. They are then uploaded + * (E2EE) to remote, and the relevant bits also saved locally in the ML DB for + * subsequent lookup or clustering. * * - Remote embeddings are fetched by subsequent clients to avoid them having to - * reindex (indexing faces is a costly operation, esp for mobile clients). + * reindex (indexing faces is a costly operation, esp. for mobile clients). * * In both these scenarios (whether generated locally or fetched from remote), - * we end up with an face index described by this {@link FaceIndex} interface. + * we end up with an face index described by this {@link RemoteFaceIndex} + * interface. We slightly prune this when saving it to the local DB, and code + * that runs locally only has access to the {@link LocalFaceIndex}. * * It has a top level envelope with information about the file (in particular * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with - * metadata about the indexing, and an array of {@link faces} each containing - * the result of a face detection and an embedding for that detected face. + * metadata about the indexing, and an . * * The word embedding is used to refer two things: The last one (faceEmbedding > * faces > embedding) is the "actual" embedding, but sometimes we colloquially * refer to the inner envelope (the "faceEmbedding") also an embedding since a * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". */ -export interface FaceIndex { +export interface RemoteFaceIndex { + /** + * An integral version number of the indexing algorithm / pipeline. + * + * Clients agree out of band what a particular version means, and guarantee + * that an embedding with a particular version will be the same (to epsilon + * cosine similarity) irrespective of the client that indexed the file. + */ + version: number; + /** + * The UA for the client which generated this embedding. + */ + client: string; + /** + * The list of faces (and their embeddings) detected in the file. + * + * Each of the items is a {@link Face}, containing the result of a face + * detection, and an embedding for that detected face. + */ + faces: Face[]; +} + +/** + * A pruned version of {@link RemoteFaceIndex} with data and shape that are + * suitable for local persistence and usage. + */ +export type LocalFaceIndex = Omit & { /** * The ID of the {@link EnteFile} whose index this is. * @@ -69,36 +99,7 @@ export interface FaceIndex { * user will get a file entry with a fileID unique to them). */ fileID: number; - /** - * The width (in px) of the image (file). - */ - width: number; - /** - * The height (in px) of the image (file). - */ - height: number; - /** - * The "face embedding" for the file. - * - * This is an envelope that contains a list of indexed faces and metadata - * about the indexing. - */ - faceEmbedding: { - /** - * An integral version number of the indexing algorithm / pipeline. - * - * Clients agree out of band what a particular version means. The - * guarantee is that an embedding with a particular version will be the - * same (to negligible floating point epsilons) irrespective of the - * client that indexed the file. - */ - version: number; - /** The UA for the client which generated this embedding. */ - client: string; - /** The list of faces (and their embeddings) detected in the file. */ - faces: Face[]; - }; -} +}; /** * A face detected in a file, and an embedding for this detected face.