WIP refactor data structure

web/packages/new/photos/services/ml/clip.ts

@@ -14,7 +14,7 @@ export const clipIndexingVersion = 1;
 /**
  * The CLIP embedding for a file (and some metadata).
  *
- * See {@link FaceIndex} for a similar structure with more comprehensive
+ * See {@link RemoteFaceIndex} for a similar structure with more comprehensive
  * documentation.
  *
  * ---

web/packages/new/photos/services/ml/crop.ts

@@ -2,7 +2,7 @@ import { blobCache } from "@/next/blob-cache";
 import { ensure } from "@/utils/ensure";
 import type { EnteFile } from "../../types/file";
 import { renderableEnteFileBlob } from "./blob";
-import { type Box, type FaceIndex } from "./face";
+import { type Box, type LocalFaceIndex } from "./face";
 import { clamp } from "./math";
 
 /**
@@ -16,15 +16,15 @@ import { clamp } from "./math";
  *
  * @param enteFile The {@link EnteFile} whose face crops we want to generate.
  *
- * @param faceIndex The {@link FaceIndex} containing information about the faces
- * detected in the given image.
+ * @param faces The {@link LocalFaceIndex} containing information about the
+ * faces detected in the given image.
  *
  * The generated face crops are saved in a local cache and can subsequently be
  * retrieved from the {@link BlobCache} named "face-crops".
  */
 export const regenerateFaceCrops = async (
     enteFile: EnteFile,
-    faceIndex: FaceIndex,
+    faces: Face[],
 ) => {
     const imageBitmap = await createImageBitmap(
         await renderableEnteFileBlob(enteFile),
@@ -51,7 +51,7 @@ export const regenerateFaceCrops = async (
  */
 export const saveFaceCrops = async (
     imageBitmap: ImageBitmap,
-    faceIndex: FaceIndex,
+    faceIndex: LocalFaceIndex,
 ) => {
     const cache = await blobCache("face-crops");
 

web/packages/new/photos/services/ml/face.ts

@@ -36,30 +36,60 @@ export const faceIndexingVersion = 1;
 /**
  * The faces in a file (and an embedding for each of them).
  *
- * This interface describes the format of both local and remote face data.
+ * This interface describes the format of remote face related data (aka "face
+ * index") for a file. See {@link LocalFaceIndex} for the subset of the fields
+ * that we also persist locally.
  *
  * - Local face detections and embeddings (collectively called as the face
- *   index) are generated by the current client when uploading a file (or when
- *   noticing a file which doesn't yet have a face index), stored in the local
- *   IndexedDB ("ml/db") and also uploaded (E2EE) to remote.
+ *   index) are generated by the current client when uploading a file, or when
+ *   noticing a file which doesn't yet have a face index. They are then uploaded
+ *   (E2EE) to remote, and the relevant bits also saved locally in the ML DB for
+ *   subsequent lookup or clustering.
  *
  * - Remote embeddings are fetched by subsequent clients to avoid them having to
- *   reindex (indexing faces is a costly operation, esp for mobile clients).
+ *   reindex (indexing faces is a costly operation, esp. for mobile clients).
  *
  * In both these scenarios (whether generated locally or fetched from remote),
- * we end up with an face index described by this {@link FaceIndex} interface.
+ * we end up with an face index described by this {@link RemoteFaceIndex}
+ * interface. We slightly prune this when saving it to the local DB, and code
+ * that runs locally only has access to the {@link LocalFaceIndex}.
  *
  * It has a top level envelope with information about the file (in particular
  * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with
- * metadata about the indexing, and an array of {@link faces} each containing
- * the result of a face detection and an embedding for that detected face.
+ * metadata about the indexing, and an .
  *
  * The word embedding is used to refer two things: The last one (faceEmbedding >
  * faces > embedding) is the "actual" embedding, but sometimes we colloquially
  * refer to the inner envelope (the "faceEmbedding") also an embedding since a
  * file can have other types of embedding (envelopes), e.g. a "clipEmbedding".
  */
-export interface FaceIndex {
+export interface RemoteFaceIndex {
+    /**
+     * An integral version number of the indexing algorithm / pipeline.
+     *
+     * Clients agree out of band what a particular version means, and guarantee
+     * that an embedding with a particular version will be the same (to epsilon
+     * cosine similarity) irrespective of the client that indexed the file.
+     */
+    version: number;
+    /**
+     * The UA for the client which generated this embedding.
+     */
+    client: string;
+    /**
+     * The list of faces (and their embeddings) detected in the file.
+     *
+     * Each of the items is a {@link Face}, containing the result of a face
+     * detection, and an embedding for that detected face.
+     */
+    faces: Face[];
+}
+
+/**
+ * A pruned version of {@link RemoteFaceIndex} with data and shape that are
+ * suitable for local persistence and usage.
+ */
+export type LocalFaceIndex = Omit<RemoteFaceIndex, "version" | "client"> & {
     /**
      * The ID of the {@link EnteFile} whose index this is.
      *
@@ -69,36 +99,7 @@ export interface FaceIndex {
      * user will get a file entry with a fileID unique to them).
      */
     fileID: number;
-    /**
-     * The width (in px) of the image (file).
-     */
-    width: number;
-    /**
-     * The height (in px) of the image (file).
-     */
-    height: number;
-    /**
-     * The "face embedding" for the file.
-     *
-     * This is an envelope that contains a list of indexed faces and metadata
-     * about the indexing.
-     */
-    faceEmbedding: {
-        /**
-         * An integral version number of the indexing algorithm / pipeline.
-         *
-         * Clients agree out of band what a particular version means. The
-         * guarantee is that an embedding with a particular version will be the
-         * same (to negligible floating point epsilons) irrespective of the
-         * client that indexed the file.
-         */
-        version: number;
-        /** The UA for the client which generated this embedding. */
-        client: string;
-        /** The list of faces (and their embeddings) detected in the file. */
-        faces: Face[];
-    };
-}
+};
 
 /**
  * A face detected in a file, and an embedding for this detected face.