From bae4c65ab3a2ae003c8ba9bc544a2b2d5406f66c Mon Sep 17 00:00:00 2001
From: Manav Rathi <manav@mrmr.io>
Date: Thu, 30 May 2024 09:43:22 +0530
Subject: [PATCH] Pull out the alignment

---
 web/apps/photos/src/services/face/crop.ts     | 11 ++++--
 web/apps/photos/src/services/face/f-index.ts  | 37 +++++++++++++------
 .../photos/src/services/face/types-old.ts     | 19 ----------
 3 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts
index faf7f0ac9b..8a09c1b8e6 100644
--- a/web/apps/photos/src/services/face/crop.ts
+++ b/web/apps/photos/src/services/face/crop.ts
@@ -1,9 +1,14 @@
 import { blobCache } from "@/next/blob-cache";
+import type { FaceAlignment } from "./f-index";
 import type { Box } from "./types";
-import type { Face, FaceAlignment } from "./types-old";
+import type { Face } from "./types-old";
 
-export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => {
-    const faceCrop = extractFaceCrop(imageBitmap, face.alignment);
+export const saveFaceCrop = async (
+    imageBitmap: ImageBitmap,
+    face: Face,
+    alignment: FaceAlignment,
+) => {
+    const faceCrop = extractFaceCrop(imageBitmap, alignment);
     const blob = await imageBitmapToBlob(faceCrop);
     faceCrop.close();
 
diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts
index a1a614f138..714f89c09f 100644
--- a/web/apps/photos/src/services/face/f-index.ts
+++ b/web/apps/photos/src/services/face/f-index.ts
@@ -22,12 +22,7 @@ import {
     warpAffineFloat32List,
 } from "./image";
 import type { Box, Dimensions } from "./types";
-import type {
-    Face,
-    FaceAlignment,
-    FaceDetection,
-    MlFileData,
-} from "./types-old";
+import type { Face, FaceDetection, MlFileData } from "./types-old";
 
 /**
  * Index faces in the given file.
@@ -109,11 +104,12 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => {
         const alignments: FaceAlignment[] = [];
 
         for (const face of mlFile.faces) {
-            const alignment = faceAlignment(face.detection);
-            face.alignment = alignment;
+            const alignment = computeFaceAlignment(face.detection);
             alignments.push(alignment);
 
-            await saveFaceCrop(imageBitmap, face);
+            // This step is not really part of the indexing pipeline, we just do
+            // it here since we have already computed the face alignment.
+            await saveFaceCrop(imageBitmap, face, alignment);
         }
 
         const alignedFacesData = convertToMobileFaceNetInput(
@@ -393,13 +389,30 @@ const makeFaceID = (
     return [`${fileID}`, xMin, yMin, xMax, yMax].join("_");
 };
 
+export interface FaceAlignment {
+    /**
+     * An affine transformation matrix (rotation, translation, scaling) to align
+     * the face extracted from the image.
+     */
+    affineMatrix: number[][];
+    /**
+     * The bounding box of the transformed box.
+     *
+     * The affine transformation shifts the original detection box a new,
+     * transformed, box (possibily rotated). This property is the bounding box
+     * of that transformed box. It is in the coordinate system of the original,
+     * full, image on which the detection occurred.
+     */
+    boundingBox: Box;
+}
+
 /**
  * Compute and return an {@link FaceAlignment} for the given face detection.
  *
  * @param faceDetection A geometry indicating a face detected in an image.
  */
-const faceAlignment = (faceDetection: FaceDetection): FaceAlignment =>
-    faceAlignmentUsingSimilarityTransform(
+const computeFaceAlignment = (faceDetection: FaceDetection): FaceAlignment =>
+    computeFaceAlignmentUsingSimilarityTransform(
         faceDetection,
         normalizeLandmarks(idealMobileFaceNetLandmarks, mobileFaceNetFaceSize),
     );
@@ -422,7 +435,7 @@ const normalizeLandmarks = (
 ): [number, number][] =>
     landmarks.map(([x, y]) => [x / faceSize, y / faceSize]);
 
-const faceAlignmentUsingSimilarityTransform = (
+const computeFaceAlignmentUsingSimilarityTransform = (
     faceDetection: FaceDetection,
     alignedLandmarks: [number, number][],
 ): FaceAlignment => {
diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts
index c64fc88ee7..ad6ef68743 100644
--- a/web/apps/photos/src/services/face/types-old.ts
+++ b/web/apps/photos/src/services/face/types-old.ts
@@ -7,29 +7,10 @@ export interface FaceDetection {
     probability?: number;
 }
 
-export interface FaceAlignment {
-    /**
-     * An affine transformation matrix (rotation, translation, scaling) to align
-     * the face extracted from the image.
-     */
-    affineMatrix: number[][];
-    /**
-     * The bounding box of the transformed box.
-     *
-     * The affine transformation shifts the original detection box a new,
-     * transformed, box (possibily rotated). This property is the bounding box
-     * of that transformed box. It is in the coordinate system of the original,
-     * full, image on which the detection occurred.
-     */
-    boundingBox: Box;
-}
-
 export interface Face {
     fileId: number;
     detection: FaceDetection;
     id: string;
-
-    alignment?: FaceAlignment;
     blurValue?: number;
 
     embedding?: Float32Array;