From b04538ada2e67ae4e590bd75c6791ae5b10620af Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 4 Jul 2024 09:57:31 +0530 Subject: [PATCH] Rename and merge --- .../photos/src/services/embeddingService.ts | 2 +- web/packages/new/photos/services/ml/crop.ts | 3 +- web/packages/new/photos/services/ml/db.ts | 2 +- .../new/photos/services/ml/embedding.ts | 3 +- .../services/ml/{index-face.ts => face.ts} | 167 +++++++++++++++++- web/packages/new/photos/services/ml/index.ts | 6 +- web/packages/new/photos/services/ml/types.ts | 159 ----------------- web/packages/new/photos/services/ml/worker.ts | 19 +- 8 files changed, 175 insertions(+), 186 deletions(-) rename web/packages/new/photos/services/ml/{index-face.ts => face.ts} (81%) delete mode 100644 web/packages/new/photos/services/ml/types.ts diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 76dd872fae..ca1e0dd8a9 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -3,7 +3,7 @@ import { getLocalTrashedFiles, } from "@/new/photos/services/files"; import type { EmbeddingModel } from "@/new/photos/services/ml/embedding"; -import type { FaceIndex } from "@/new/photos/services/ml/types"; +import type { FaceIndex } from "@/new/photos/services/ml/face"; import { EnteFile } from "@/new/photos/types/file"; import { inWorker } from "@/next/env"; import log from "@/next/log"; diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.ts index e306d9399d..17a6b3dcad 100644 --- a/web/packages/new/photos/services/ml/crop.ts +++ b/web/packages/new/photos/services/ml/crop.ts @@ -1,7 +1,6 @@ -import type { Box } from "@/new/photos/services/ml/types"; import { blobCache } from "@/next/blob-cache"; import { ensure } from "@/utils/ensure"; -import type { FaceAlignment } from "./index-face"; +import type { Box, FaceAlignment } from "./face"; /** * Return the rectangle ("crop") of the original image that contains the face diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index 91cee18f6f..b8bcad5ed9 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -1,6 +1,6 @@ import log from "@/next/log"; import { deleteDB, openDB, type DBSchema } from "idb"; -import type { FaceIndex } from "./types"; +import type { FaceIndex } from "./face"; /** * Face DB schema. diff --git a/web/packages/new/photos/services/ml/embedding.ts b/web/packages/new/photos/services/ml/embedding.ts index fdbc2ae24c..16d1cf06fb 100644 --- a/web/packages/new/photos/services/ml/embedding.ts +++ b/web/packages/new/photos/services/ml/embedding.ts @@ -13,8 +13,7 @@ import log from "@/next/log"; import { apiURL } from "@/next/origins"; import { z } from "zod"; import { saveFaceIndex } from "./db"; -import { faceIndexingVersion } from "./index-face"; -import { type FaceIndex } from "./types"; +import { type FaceIndex, faceIndexingVersion } from "./face"; /** * The embeddings that we (the current client) knows how to handle. diff --git a/web/packages/new/photos/services/ml/index-face.ts b/web/packages/new/photos/services/ml/face.ts similarity index 81% rename from web/packages/new/photos/services/ml/index-face.ts rename to web/packages/new/photos/services/ml/face.ts index aeea87e041..0c91dfe84b 100644 --- a/web/packages/new/photos/services/ml/index-face.ts +++ b/web/packages/new/photos/services/ml/face.ts @@ -10,12 +10,7 @@ import { FILE_TYPE } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; import DownloadManager from "@/new/photos/services/download"; -import type { - Box, - Dimensions, - Face, - Point, -} from "@/new/photos/services/ml/types"; + import type { EnteFile } from "@/new/photos/types/file"; import { getRenderableImage } from "@/new/photos/utils/file"; import log from "@/next/log"; @@ -43,6 +38,166 @@ import { */ export const faceIndexingVersion = 1; +/** + * The faces in a file (and an embedding for each of them). + * + * This interface describes the format of both local and remote face data. + * + * - Local face detections and embeddings (collectively called as the face + * index) are generated by the current client when uploading a file (or when + * noticing a file which doesn't yet have a face index), stored in the local + * IndexedDB ("ml/db") and also uploaded (E2EE) to remote. + * + * - Remote embeddings are fetched by subsequent clients to avoid them having to + * reindex (indexing faces is a costly operation, esp for mobile clients). + * + * In both these scenarios (whether generated locally or fetched from remote), + * we end up with an face index described by this {@link FaceIndex} interface. + * + * It has a top level envelope with information about the file (in particular + * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with + * metadata about the indexing, and an array of {@link faces} each containing + * the result of a face detection and an embedding for that detected face. + * + * The word embedding is used to refer two things: The last one (faceEmbedding > + * faces > embedding) is the "actual" embedding, but sometimes we colloquially + * refer to the inner envelope (the "faceEmbedding") also an embedding since a + * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". + */ +export interface FaceIndex { + /** + * The ID of the {@link EnteFile} whose index this is. + * + * This is used as the primary key when storing the index locally (An + * {@link EnteFile} is guaranteed to have its fileID be unique in the + * namespace of the user. Even if someone shares a file with the user the + * user will get a file entry with a fileID unique to them). + */ + fileID: number; + /** + * The width (in px) of the image (file). + */ + width: number; + /** + * The height (in px) of the image (file). + */ + height: number; + /** + * The "face embedding" for the file. + * + * This is an envelope that contains a list of indexed faces and metadata + * about the indexing. + */ + faceEmbedding: { + /** + * An integral version number of the indexing algorithm / pipeline. + * + * Clients agree out of band what a particular version means. The + * guarantee is that an embedding with a particular version will be the + * same (to negligible floating point epsilons) irrespective of the + * client that indexed the file. + */ + version: number; + /** The UA for the client which generated this embedding. */ + client: string; + /** The list of faces (and their embeddings) detected in the file. */ + faces: Face[]; + }; +} + +/** + * A face detected in a file, and an embedding for this detected face. + * + * During face indexing, we first detect all the faces in a particular file. + * Then for each such detected region, we compute an embedding of that part of + * the file. Together, this detection region and the emedding travel together in + * this {@link Face} interface. + */ +export interface Face { + /** + * A unique identifier for the face. + * + * This ID is guaranteed to be unique for all the faces detected in all the + * files for the user. In particular, each file can have multiple faces but + * they all will get their own unique {@link faceID}. + */ + faceID: string; + /** + * The face detection. Describes the region within the image that was + * detected to be a face, and a set of landmarks (e.g. "eyes") of the + * detection. + * + * All coordinates are relative to and normalized by the image's dimension, + * i.e. they have been normalized to lie between 0 and 1, with 0 being the + * left (or top) and 1 being the width (or height) of the image. + */ + detection: { + /** + * The region within the image that contains the face. + * + * All coordinates and sizes are between 0 and 1, normalized by the + * dimensions of the image. + * */ + box: Box; + /** + * Face "landmarks", e.g. eyes. + * + * The exact landmarks and their order depends on the face detection + * algorithm being used. + * + * The coordinatesare between 0 and 1, normalized by the dimensions of + * the image. + */ + landmarks: Point[]; + }; + /** + * An correctness probability (0 to 1) that the face detection algorithm + * gave to the detection. Higher values are better. + */ + score: number; + /** + * The computed blur for the detected face. + * + * The exact semantics and range for these (floating point) values depend on + * the face indexing algorithm / pipeline version being used. + * */ + blur: number; + /** + * An embedding for the face. + * + * This is an opaque numeric (signed floating point) vector whose semantics + * and length depend on the version of the face indexing algorithm / + * pipeline that we are using. However, within a set of embeddings with the + * same version, the property is that two such embedding vectors will be + * "cosine similar" to each other if they are both faces of the same person. + */ + embedding: number[]; +} + +/** The x and y coordinates of a point. */ +export interface Point { + x: number; + y: number; +} + +/** The dimensions of something, say an image. */ +export interface Dimensions { + width: number; + height: number; +} + +/** A rectangle given by its top left coordinates and dimensions. */ +export interface Box { + /** The x coordinate of the the top left (xMin). */ + x: number; + /** The y coodinate of the top left (yMin). */ + y: number; + /** The width of the box. */ + width: number; + /** The height of the box. */ + height: number; +} + /** * Index faces in the given file. * diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index e0e92212d9..e3c2b616b4 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -7,16 +7,12 @@ import { isBetaUser, isInternalUser, } from "@/new/photos/services/feature-flags"; -import { - clearFaceDB, - faceIndex, - indexableAndIndexedCounts, -} from "@/new/photos/services/ml/db"; import type { EnteFile } from "@/new/photos/types/file"; import { clientPackageName, isDesktop } from "@/next/app"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { clearFaceDB, faceIndex, indexableAndIndexedCounts } from "./db"; import { MLWorker } from "./worker"; /** diff --git a/web/packages/new/photos/services/ml/types.ts b/web/packages/new/photos/services/ml/types.ts deleted file mode 100644 index 1d3168d58b..0000000000 --- a/web/packages/new/photos/services/ml/types.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * The faces in a file (and an embedding for each of them). - * - * This interface describes the format of both local and remote face data. - * - * - Local face detections and embeddings (collectively called as the face - * index) are generated by the current client when uploading a file (or when - * noticing a file which doesn't yet have a face index), stored in the local - * IndexedDB ("ml/db") and also uploaded (E2EE) to remote. - * - * - Remote embeddings are fetched by subsequent clients to avoid them having to - * reindex (indexing faces is a costly operation, esp for mobile clients). - * - * In both these scenarios (whether generated locally or fetched from remote), - * we end up with an face index described by this {@link FaceIndex} interface. - * - * It has a top level envelope with information about the file (in particular - * the primary key {@link fileID}), an inner envelope {@link faceEmbedding} with - * metadata about the indexing, and an array of {@link faces} each containing - * the result of a face detection and an embedding for that detected face. - * - * The word embedding is used to refer two things: The last one (faceEmbedding > - * faces > embedding) is the "actual" embedding, but sometimes we colloquially - * refer to the inner envelope (the "faceEmbedding") also an embedding since a - * file can have other types of embedding (envelopes), e.g. a "clipEmbedding". - */ -export interface FaceIndex { - /** - * The ID of the {@link EnteFile} whose index this is. - * - * This is used as the primary key when storing the index locally (An - * {@link EnteFile} is guaranteed to have its fileID be unique in the - * namespace of the user. Even if someone shares a file with the user the - * user will get a file entry with a fileID unique to them). - */ - fileID: number; - /** - * The width (in px) of the image (file). - */ - width: number; - /** - * The height (in px) of the image (file). - */ - height: number; - /** - * The "face embedding" for the file. - * - * This is an envelope that contains a list of indexed faces and metadata - * about the indexing. - */ - faceEmbedding: { - /** - * An integral version number of the indexing algorithm / pipeline. - * - * Clients agree out of band what a particular version means. The - * guarantee is that an embedding with a particular version will be the - * same (to negligible floating point epsilons) irrespective of the - * client that indexed the file. - */ - version: number; - /** The UA for the client which generated this embedding. */ - client: string; - /** The list of faces (and their embeddings) detected in the file. */ - faces: Face[]; - }; -} - -/** - * A face detected in a file, and an embedding for this detected face. - * - * During face indexing, we first detect all the faces in a particular file. - * Then for each such detected region, we compute an embedding of that part of - * the file. Together, this detection region and the emedding travel together in - * this {@link Face} interface. - */ -export interface Face { - /** - * A unique identifier for the face. - * - * This ID is guaranteed to be unique for all the faces detected in all the - * files for the user. In particular, each file can have multiple faces but - * they all will get their own unique {@link faceID}. - */ - faceID: string; - /** - * The face detection. Describes the region within the image that was - * detected to be a face, and a set of landmarks (e.g. "eyes") of the - * detection. - * - * All coordinates are relative to and normalized by the image's dimension, - * i.e. they have been normalized to lie between 0 and 1, with 0 being the - * left (or top) and 1 being the width (or height) of the image. - */ - detection: { - /** - * The region within the image that contains the face. - * - * All coordinates and sizes are between 0 and 1, normalized by the - * dimensions of the image. - * */ - box: Box; - /** - * Face "landmarks", e.g. eyes. - * - * The exact landmarks and their order depends on the face detection - * algorithm being used. - * - * The coordinatesare between 0 and 1, normalized by the dimensions of - * the image. - */ - landmarks: Point[]; - }; - /** - * An correctness probability (0 to 1) that the face detection algorithm - * gave to the detection. Higher values are better. - */ - score: number; - /** - * The computed blur for the detected face. - * - * The exact semantics and range for these (floating point) values depend on - * the face indexing algorithm / pipeline version being used. - * */ - blur: number; - /** - * An embedding for the face. - * - * This is an opaque numeric (signed floating point) vector whose semantics - * and length depend on the version of the face indexing algorithm / - * pipeline that we are using. However, within a set of embeddings with the - * same version, the property is that two such embedding vectors will be - * "cosine similar" to each other if they are both faces of the same person. - */ - embedding: number[]; -} - -/** The x and y coordinates of a point. */ -export interface Point { - x: number; - y: number; -} - -/** The dimensions of something, say an image. */ -export interface Dimensions { - width: number; - height: number; -} - -/** A rectangle given by its top left coordinates and dimensions. */ -export interface Box { - /** The x coordinate of the the top left (xMin). */ - x: number; - /** The y coodinate of the top left (yMin). */ - y: number; - /** The width of the box. */ - width: number; - /** The height of the box. */ - height: number; -} diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 1ad874d74c..535b659a5a 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -1,22 +1,21 @@ -import downloadManager from "@/new/photos/services/download"; -import { - indexableFileIDs, - markIndexingFailed, - saveFaceIndex, - updateAssumingLocalFiles, -} from "@/new/photos/services/ml/db"; -import type { FaceIndex } from "@/new/photos/services/ml/types"; import type { EnteFile } from "@/new/photos/types/file"; +import { fileLogID } from "@/new/photos/utils/file"; import { getKVN } from "@/next/kv"; import { ensureAuthToken } from "@/next/local-user"; import log from "@/next/log"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { expose } from "comlink"; -import { fileLogID } from "../../utils/file"; +import downloadManager from "../download"; import { getAllLocalFiles, getLocalTrashedFiles } from "../files"; +import { + indexableFileIDs, + markIndexingFailed, + saveFaceIndex, + updateAssumingLocalFiles, +} from "./db"; import { pullFaceEmbeddings, putFaceIndex } from "./embedding"; -import { indexFaces } from "./index-face"; +import { type FaceIndex, indexFaces } from "./face"; const idleDurationStart = 5; /* 5 seconds */ const idleDurationMax = 16 * 60; /* 16 minutes */