Restore width and height

..(discussed)

web/packages/new/photos/services/ml/embedding.ts

@@ -108,8 +108,6 @@ const RemoteEmbedding = z.object({
      * the crypto layer.
      */
     decryptionHeader: z.string(),
-    /** Last time (epoch ms) this embedding was updated. */
-    updatedAt: z.number(),
 });
 
 type RemoteEmbedding = z.infer<typeof RemoteEmbedding>;
@@ -339,6 +337,23 @@ const putEmbeddingString = async (
 
 // MARK: - Combined
 
+/**
+ * The decrypted payload of a {@link RemoteEmbedding} for the "combined"
+ * {@link EmbeddingModel}.
+ *
+ * [Note: Preserve unknown derived data fields]
+ *
+ * There is one entry for each of the embedding types that the current client
+ * knows about. However, there might be other fields apart from the known ones
+ * at the top level, and we need to ensure that we preserve them verbatim when
+ * trying use {@link putDerivedData} with an {@link RemoteDerivedData} obtained
+ * from remote as the base, with locally indexed additions.
+ */
+export type RemoteDerivedData = Record<string, unknown> & {
+    face: RemoteFaceIndex;
+    clip: RemoteCLIPIndex;
+};
+
 /**
  * Update the combined derived data stored for given {@link enteFile} on remote.
  * This allows other clients to directly pull the derived data instead of
@@ -347,24 +362,14 @@ const putEmbeddingString = async (
  * The data on remote will be replaced unconditionally, and it is up to the
  * client (us) to ensure that we preserve the parts of the pre-existing derived
  * data (if any) that we did not understand or touch.
+ *
+ * See: [Note: Preserve unknown derived data fields].
  */
 export const putDerivedData = async (
     enteFile: EnteFile,
-    remoteFaceIndex: RemoteFaceIndex,
-    remoteCLIPIndex: RemoteCLIPIndex,
-) => {
-    const combined = {
-        face: remoteFaceIndex,
-        clip: remoteCLIPIndex,
-    };
-    log.debug(() => ["Uploading derived data", combined]);
-
-    return putEmbedding(
-        enteFile,
-        "combined",
-        await gzip(JSON.stringify(combined)),
-    );
-};
+    derivedData: RemoteDerivedData,
+) =>
+    putEmbedding(enteFile, "combined", await gzip(JSON.stringify(derivedData)));
 
 /**
  * Compress the given {@link string} using "gzip" and return the resultant

web/packages/new/photos/services/ml/face.ts

@@ -55,6 +55,18 @@ export const faceIndexingVersion = 1;
  * and {@link RemoteFaceIndex} types respectively.
  */
 export interface FaceIndex {
+    /**
+     * The width (in px) of the image (file).
+     *
+     * Having the image dimensions here is useful since the coordinates inside
+     * the {@link Face}s are all normalized (0 to 1) to the width and height of
+     * the image.
+     */
+    width: number;
+    /**
+     * The height (in px) of the image (file).
+     */
+    height: number;
     /**
      * The list of faces (and their embeddings) detected in the file.
      *
@@ -211,6 +223,8 @@ export const indexFaces = async (
     { data: imageData }: ImageBitmapAndData,
     electron: MLWorkerElectron,
 ): Promise<FaceIndex> => ({
+    width: imageData.width,
+    height: imageData.height,
     faces: await indexFaces_(enteFile.id, imageData, electron),
 });
 

web/packages/new/photos/services/ml/worker.ts

@@ -25,7 +25,7 @@ import {
     saveFaceIndex,
     updateAssumingLocalFiles,
 } from "./db";
-import { pullFaceEmbeddings, putDerivedData } from "./embedding";
+import { putDerivedData } from "./embedding";
 import { faceIndexingVersion, indexFaces, type FaceIndex } from "./face";
 import type { MLWorkerDelegate, MLWorkerElectron } from "./worker-types";
 
@@ -208,32 +208,6 @@ export class MLWorker {
 
 expose(MLWorker);
 
-/**
- * Pull embeddings from remote.
- *
- * Return true atleast one embedding was pulled.
- */
-const pull = async () => {
-    const res = await Promise.allSettled([
-        pullFaceEmbeddings(),
-        // TODO-ML: clip-test
-        // pullCLIPEmbeddings(),
-    ]);
-    for (const r of res) {
-        switch (r.status) {
-            case "fulfilled":
-                // Return true if any pulled something.
-                if (r.value) return true;
-                break;
-            case "rejected":
-                // Throw if any failed.
-                throw r.reason;
-        }
-    }
-    // Return false if neither pulled anything.
-    return false;
-};
-
 /**
  * Find out files which need to be indexed. Then index the next batch of them.
  *
@@ -416,20 +390,23 @@ const index = async (
             return `Indexed ${nf} faces and clip in ${f} (${ms} ms)`;
         });
 
-        const remoteFaceIndex = {
-            ...faceIndex,
-            version: faceIndexingVersion,
-            client: userAgent,
+        const derivedData = {
+            face: {
+                version: faceIndexingVersion,
+                client: userAgent,
+                ...faceIndex,
+            },
+            clip: {
+                version: clipIndexingVersion,
+                client: userAgent,
+                ...clipIndex,
+            },
         };
 
-        const remoteCLIPIndex = {
-            ...clipIndex,
-            version: clipIndexingVersion,
-            client: userAgent,
-        };
+        log.debug(() => ["Uploading derived data", derivedData]);
 
         try {
-            await putDerivedData(enteFile, remoteFaceIndex, remoteCLIPIndex);
+            await putDerivedData(enteFile, derivedData);
         } catch (e) {
             // See: [Note: Transient and permanent indexing failures]
             log.error(`Failed to put face index for ${f}`, e);