From 0a9f7b8635ed586eaf1303f1b6edfd1df7df5d17 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 09:22:55 +0530 Subject: [PATCH 01/47] Remove unused --- .../photos/src/services/machineLearning/mlWorkManager.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index d696e883f7..b1df495fae 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -18,11 +18,6 @@ const LOCAL_FILES_UPDATED_DEBOUNCE_SEC = 30; export type JobState = "Scheduled" | "Running" | "NotScheduled"; -export interface JobConfig { - intervalSec: number; - backoffMultiplier: number; -} - export interface MLSyncJobResult { shouldBackoff: boolean; mlSyncResult: MLSyncResult; From 160e3609411775d874f10710054f54788f1198ba Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 09:25:10 +0530 Subject: [PATCH 02/47] Inline --- .../src/services/machineLearning/mlWorkManager.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index b1df495fae..6ce5d80084 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -12,10 +12,6 @@ import { MLSyncResult } from "services/ml/types"; import { EnteFile } from "types/file"; import { logQueueStats } from "./machineLearningService"; -const LIVE_SYNC_IDLE_DEBOUNCE_SEC = 30; -const LIVE_SYNC_QUEUE_TIMEOUT_SEC = 300; -const LOCAL_FILES_UPDATED_DEBOUNCE_SEC = 30; - export type JobState = "Scheduled" | "Running" | "NotScheduled"; export interface MLSyncJobResult { @@ -113,18 +109,18 @@ class MLWorkManager { this.liveSyncQueue = new PQueue({ concurrency: 1, // TODO: temp, remove - timeout: LIVE_SYNC_QUEUE_TIMEOUT_SEC * 1000, + timeout: 300 * 1000, throwOnTimeout: true, }); this.mlSearchEnabled = false; this.debouncedLiveSyncIdle = debounce( () => this.onLiveSyncIdle(), - LIVE_SYNC_IDLE_DEBOUNCE_SEC * 1000, + 30 * 1000, ); this.debouncedFilesUpdated = debounce( () => this.mlSearchEnabled && this.localFilesUpdatedHandler(), - LOCAL_FILES_UPDATED_DEBOUNCE_SEC * 1000, + 30 * 1000, ); } From 0f008035197b64beb2f4ca89c85fd23bc08e5d85 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 09:33:41 +0530 Subject: [PATCH 03/47] Inline --- .../machineLearning/machineLearningService.ts | 49 +++++++++++++------ web/apps/photos/src/services/ml/db.ts | 9 ++-- web/apps/photos/src/services/searchService.ts | 5 +- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index d89fdebc94..067a91d2db 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -11,10 +11,7 @@ import PQueue from "p-queue"; import downloadManager from "services/download"; import { putEmbedding } from "services/embeddingService"; import { getLocalFiles } from "services/fileService"; -import mlIDbStorage, { - ML_SEARCH_CONFIG_NAME, - ML_SYNC_CONFIG_NAME, -} from "services/ml/db"; +import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/ml/db"; import { BlurDetectionMethod, BlurDetectionService, @@ -52,7 +49,13 @@ import PeopleService from "./peopleService"; import ReaderService from "./readerService"; import yoloFaceDetectionService from "./yoloFaceDetectionService"; -export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { +/** + * TODO-ML(MR): What and why. + * Also, needs to be 1 (in sync with mobile) when we move out of beta. + */ +export const defaultMLVersion = 3; + +const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { batchSize: 200, imageSource: "Original", faceDetection: { @@ -90,7 +93,7 @@ export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { // maxDistanceInsideCluster: 0.4, generateDebugInfo: true, }, - mlVersion: 3, + mlVersion: defaultMLVersion, }; export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { @@ -99,10 +102,6 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { export const MAX_ML_SYNC_ERROR_COUNT = 1; -export async function getMLSyncConfig() { - return mlIDbStorage.getConfig(ML_SYNC_CONFIG_NAME, DEFAULT_ML_SYNC_CONFIG); -} - export async function getMLSearchConfig() { if (isInternalUserForML()) { return mlIDbStorage.getConfig( @@ -481,9 +480,18 @@ class MachineLearningService { if (!this.syncContext) { log.info("Creating syncContext"); - this.syncContext = getMLSyncConfig().then((mlSyncConfig) => - MLFactory.getMLSyncContext(token, userID, mlSyncConfig, true), - ); + const mlSyncConfig = DEFAULT_ML_SYNC_CONFIG; + // TODO-ML(MR): Keep as promise for now. + this.syncContext = new Promise((resolve) => { + resolve( + MLFactory.getMLSyncContext( + token, + userID, + mlSyncConfig, + true, + ), + ); + }); } else { log.info("reusing existing syncContext"); } @@ -493,9 +501,18 @@ class MachineLearningService { private async getLocalSyncContext(token: string, userID: number) { if (!this.localSyncContext) { log.info("Creating localSyncContext"); - this.localSyncContext = getMLSyncConfig().then((mlSyncConfig) => - MLFactory.getMLSyncContext(token, userID, mlSyncConfig, false), - ); + // TODO-ML(MR): + this.localSyncContext = new Promise((resolve) => { + const mlSyncConfig = DEFAULT_ML_SYNC_CONFIG; + resolve( + MLFactory.getMLSyncContext( + token, + userID, + mlSyncConfig, + false, + ), + ); + }); } else { log.info("reusing existing localSyncContext"); } diff --git a/web/apps/photos/src/services/ml/db.ts b/web/apps/photos/src/services/ml/db.ts index c55d38f2be..7723ebeae5 100644 --- a/web/apps/photos/src/services/ml/db.ts +++ b/web/apps/photos/src/services/ml/db.ts @@ -11,7 +11,6 @@ import { import isElectron from "is-electron"; import { DEFAULT_ML_SEARCH_CONFIG, - DEFAULT_ML_SYNC_CONFIG, MAX_ML_SYNC_ERROR_COUNT, } from "services/machineLearning/machineLearningService"; import { Face, MLLibraryData, MlFileData, Person } from "services/ml/types"; @@ -26,7 +25,6 @@ export interface IndexStatus { interface Config {} -export const ML_SYNC_CONFIG_NAME = "ml-sync"; export const ML_SEARCH_CONFIG_NAME = "ml-search"; const MLDATA_DB_NAME = "mldata"; @@ -141,10 +139,11 @@ class MLIDbStorage { DEFAULT_ML_SYNC_JOB_CONFIG, "ml-sync-job", ); - */ + await tx .objectStore("configs") .add(DEFAULT_ML_SYNC_CONFIG, ML_SYNC_CONFIG_NAME); + */ } if (oldVersion < 3) { await tx @@ -163,6 +162,10 @@ class MLIDbStorage { .objectStore("configs") .delete(ML_SEARCH_CONFIG_NAME); + await tx + .objectStore("configs") + .delete(""ml-sync""); + await tx .objectStore("configs") .delete("ml-sync-job"); diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index a212fc9dcf..54c69ee080 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,7 +2,7 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; -import { getMLSyncConfig } from "services/machineLearning/machineLearningService"; +import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import mlIDbStorage from "services/ml/db"; import { Person } from "services/ml/types"; import { Collection } from "types/collection"; @@ -175,8 +175,7 @@ export async function getAllPeopleSuggestion(): Promise> { export async function getIndexStatusSuggestion(): Promise { try { - const config = await getMLSyncConfig(); - const indexStatus = await mlIDbStorage.getIndexStatus(config.mlVersion); + const indexStatus = await mlIDbStorage.getIndexStatus(defaultMLVersion); let label; if (!indexStatus.localFilesSynced) { From 01108141c2d4309951841dc81794eec472da379f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 09:39:05 +0530 Subject: [PATCH 04/47] Inline --- .../machineLearning/machineLearningService.ts | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 067a91d2db..fd9f634f25 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -96,12 +96,12 @@ const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { mlVersion: defaultMLVersion, }; +export const MAX_ML_SYNC_ERROR_COUNT = 1; + export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { enabled: false, }; -export const MAX_ML_SYNC_ERROR_COUNT = 1; - export async function getMLSearchConfig() { if (isInternalUserForML()) { return mlIDbStorage.getConfig( @@ -240,24 +240,16 @@ export class LocalMLSyncContext implements MLSyncContext { this.config = config; this.shouldUpdateMLVersion = shouldUpdateMLVersion; - this.faceDetectionService = MLFactory.getFaceDetectionService( - this.config.faceDetection.method, - ); - this.faceCropService = MLFactory.getFaceCropService( - this.config.faceCrop.method, - ); - this.faceAlignmentService = MLFactory.getFaceAlignmentService( - this.config.faceAlignment.method, - ); - this.blurDetectionService = MLFactory.getBlurDetectionService( - this.config.blurDetection.method, - ); - this.faceEmbeddingService = MLFactory.getFaceEmbeddingService( - this.config.faceEmbedding.method, - ); - this.faceClusteringService = MLFactory.getClusteringService( - this.config.faceClustering.method, - ); + this.faceDetectionService = + MLFactory.getFaceDetectionService("YoloFace"); + this.faceCropService = MLFactory.getFaceCropService("ArcFace"); + this.faceAlignmentService = + MLFactory.getFaceAlignmentService("ArcFace"); + this.blurDetectionService = + MLFactory.getBlurDetectionService("Laplacian"); + this.faceEmbeddingService = + MLFactory.getFaceEmbeddingService("MobileFaceNet"); + this.faceClusteringService = MLFactory.getClusteringService("Hdbscan"); this.outOfSyncFiles = []; this.nSyncedFiles = 0; From dd38232836c0ad6a42206cad570e862bd2ebc372 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 10:41:05 +0530 Subject: [PATCH 05/47] Subsume --- .../services/machineLearning/faceService.ts | 11 ++++---- .../machineLearning/machineLearningService.ts | 27 +++++++------------ .../services/machineLearning/readerService.ts | 6 +++-- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index e1c71ba742..34b2ef669c 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -15,6 +15,7 @@ import ReaderService, { getFaceId, getLocalFile, } from "./readerService"; +import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; class FaceService { async syncFileFaceDetections( @@ -27,7 +28,7 @@ class FaceService { oldMlFile?.faceDetectionMethod, syncContext.faceDetectionService.method, ) && - oldMlFile?.imageSource === syncContext.config.imageSource + oldMlFile?.imageSource === "Original" ) { newMlFile.faces = oldMlFile?.faces?.map((existingFace) => ({ id: existingFace.id, @@ -223,10 +224,10 @@ class FaceService { const faceCrop = await syncContext.faceCropService.getFaceCrop( imageBitmap, face.detection, - syncContext.config.faceCrop, + DEFAULT_ML_SYNC_CONFIG.faceCrop, ); - const blobOptions = syncContext.config.faceCrop.blobOptions; + const blobOptions = DEFAULT_ML_SYNC_CONFIG.faceCrop.blobOptions; const blob = await imageBitmapToBlob(faceCrop.image, blobOptions); const cache = await openCache("face-crops"); @@ -252,7 +253,7 @@ class FaceService { ) { // await this.init(); - const clusteringConfig = syncContext.config.faceClustering; + const clusteringConfig = DEFAULT_ML_SYNC_CONFIG.faceClustering; if (!allFaces || allFaces.length < clusteringConfig.minInputSize) { log.info( @@ -266,7 +267,7 @@ class FaceService { syncContext.mlLibraryData.faceClusteringResults = await syncContext.faceClusteringService.cluster( allFaces.map((f) => Array.from(f.embedding)), - syncContext.config.faceClustering, + DEFAULT_ML_SYNC_CONFIG.faceClustering, ); syncContext.mlLibraryData.faceClusteringMethod = syncContext.faceClusteringService.method; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index fd9f634f25..8f12d60147 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -55,7 +55,9 @@ import yoloFaceDetectionService from "./yoloFaceDetectionService"; */ export const defaultMLVersion = 3; -const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { +const batchSize = 200; + +export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { batchSize: 200, imageSource: "Original", faceDetection: { @@ -186,19 +188,13 @@ export class MLFactory { config: MLSyncConfig, shouldUpdateMLVersion: boolean = true, ) { - return new LocalMLSyncContext( - token, - userID, - config, - shouldUpdateMLVersion, - ); + return new LocalMLSyncContext(token, userID, shouldUpdateMLVersion); } } export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; - public config: MLSyncConfig; public shouldUpdateMLVersion: boolean; public faceDetectionService: FaceDetectionService; @@ -231,13 +227,11 @@ export class LocalMLSyncContext implements MLSyncContext { constructor( token: string, userID: number, - config: MLSyncConfig, shouldUpdateMLVersion: boolean = true, concurrency?: number, ) { this.token = token; this.userID = userID; - this.config = config; this.shouldUpdateMLVersion = shouldUpdateMLVersion; this.faceDetectionService = @@ -318,8 +312,7 @@ class MachineLearningService { // may be need to just take synced files on latest ml version for indexing if ( syncContext.outOfSyncFiles.length <= 0 || - (syncContext.nSyncedFiles === syncContext.config.batchSize && - Math.random() < 0.2) + (syncContext.nSyncedFiles === batchSize && Math.random() < 0.2) ) { await this.syncIndex(syncContext); } @@ -425,8 +418,8 @@ class MachineLearningService { private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( - syncContext.config.batchSize, - syncContext.config.mlVersion, + batchSize, + defaultMLVersion, MAX_ML_SYNC_ERROR_COUNT, ); @@ -535,7 +528,7 @@ class MachineLearningService { localFile, ); - if (syncContext.nSyncedFiles >= syncContext.config.batchSize) { + if (syncContext.nSyncedFiles >= batchSize) { await this.closeLocalSyncContext(); } // await syncContext.dispose(); @@ -603,7 +596,7 @@ class MachineLearningService { (fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ?? this.newMlData(enteFile.id); if ( - fileContext.oldMlFile?.mlVersion === syncContext.config.mlVersion + fileContext.oldMlFile?.mlVersion === defaultMLVersion // TODO: reset mlversion of all files when user changes image source ) { return fileContext.oldMlFile; @@ -611,7 +604,7 @@ class MachineLearningService { const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); if (syncContext.shouldUpdateMLVersion) { - newMlFile.mlVersion = syncContext.config.mlVersion; + newMlFile.mlVersion = defaultMLVersion; } else if (fileContext.oldMlFile?.mlVersion) { newMlFile.mlVersion = fileContext.oldMlFile.mlVersion; } diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index 6660cf8667..e6b95d2534 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -12,6 +12,7 @@ import { import { EnteFile } from "types/file"; import { getRenderableImage } from "utils/file"; import { clamp } from "utils/image"; +import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; class ReaderService { async getImageBitmap( @@ -35,7 +36,7 @@ class ReaderService { fileContext.localFile, ); } else if ( - syncContext.config.imageSource === "Original" && + DEFAULT_ML_SYNC_CONFIG.imageSource === "Original" && [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes( fileContext.enteFile.metadata.fileType, ) @@ -49,7 +50,8 @@ class ReaderService { ); } - fileContext.newMlFile.imageSource = syncContext.config.imageSource; + fileContext.newMlFile.imageSource = + DEFAULT_ML_SYNC_CONFIG.imageSource; const { width, height } = fileContext.imageBitmap; fileContext.newMlFile.imageDimensions = { width, height }; From 095e8c7091da1798d3f43d5bdfbf113e52c3f18a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 10:42:54 +0530 Subject: [PATCH 06/47] Inline --- .../services/machineLearning/faceService.ts | 16 +++------- .../machineLearning/machineLearningService.ts | 31 ++----------------- .../services/machineLearning/readerService.ts | 11 ++----- web/apps/photos/src/services/ml/types.ts | 1 - 4 files changed, 10 insertions(+), 49 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 34b2ef669c..93e2c0a0a5 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -10,12 +10,12 @@ import { type Versioned, } from "services/ml/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; import ReaderService, { fetchImageBitmap, getFaceId, getLocalFile, } from "./readerService"; -import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; class FaceService { async syncFileFaceDetections( @@ -44,10 +44,7 @@ class FaceService { newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method; fileContext.newDetection = true; - const imageBitmap = await ReaderService.getImageBitmap( - syncContext, - fileContext, - ); + const imageBitmap = await ReaderService.getImageBitmap(fileContext); const timerId = `faceDetection-${fileContext.enteFile.id}`; console.time(timerId); const faceDetections = @@ -93,10 +90,7 @@ class FaceService { return; } - const imageBitmap = await ReaderService.getImageBitmap( - syncContext, - fileContext, - ); + const imageBitmap = await ReaderService.getImageBitmap(fileContext); newMlFile.faceCropMethod = syncContext.faceCropService.method; for (const face of newMlFile.faces) { @@ -128,7 +122,7 @@ class FaceService { fileContext.newAlignment = true; const imageBitmap = fileContext.imageBitmap || - (await ReaderService.getImageBitmap(syncContext, fileContext)); + (await ReaderService.getImageBitmap(fileContext)); // Execute the face alignment calculations for (const face of newMlFile.faces) { @@ -179,7 +173,7 @@ class FaceService { newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method; // TODO: when not storing face crops, image will be needed to extract faces // fileContext.imageBitmap || - // (await this.getImageBitmap(syncContext, fileContext)); + // (await this.getImageBitmap(fileContext)); const embeddings = await syncContext.faceEmbeddingService.getFaceEmbeddings( diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 8f12d60147..2ebd14efeb 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -181,15 +181,6 @@ export class MLFactory { throw Error("Unknon clustering method: " + method); } - - public static getMLSyncContext( - token: string, - userID: number, - config: MLSyncConfig, - shouldUpdateMLVersion: boolean = true, - ) { - return new LocalMLSyncContext(token, userID, shouldUpdateMLVersion); - } } export class LocalMLSyncContext implements MLSyncContext { @@ -465,17 +456,9 @@ class MachineLearningService { if (!this.syncContext) { log.info("Creating syncContext"); - const mlSyncConfig = DEFAULT_ML_SYNC_CONFIG; // TODO-ML(MR): Keep as promise for now. this.syncContext = new Promise((resolve) => { - resolve( - MLFactory.getMLSyncContext( - token, - userID, - mlSyncConfig, - true, - ), - ); + resolve(new LocalMLSyncContext(token, userID, true)); }); } else { log.info("reusing existing syncContext"); @@ -488,15 +471,7 @@ class MachineLearningService { log.info("Creating localSyncContext"); // TODO-ML(MR): this.localSyncContext = new Promise((resolve) => { - const mlSyncConfig = DEFAULT_ML_SYNC_CONFIG; - resolve( - MLFactory.getMLSyncContext( - token, - userID, - mlSyncConfig, - false, - ), - ); + resolve(new LocalMLSyncContext(token, userID, false)); }); } else { log.info("reusing existing localSyncContext"); @@ -610,7 +585,7 @@ class MachineLearningService { } try { - await ReaderService.getImageBitmap(syncContext, fileContext); + await ReaderService.getImageBitmap(fileContext); await Promise.all([ this.syncFileAnalyzeFaces(syncContext, fileContext), ]); diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index e6b95d2534..98052edfa4 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -4,21 +4,14 @@ import log from "@/next/log"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; import { Dimensions } from "services/ml/geom"; -import { - DetectedFace, - MLSyncContext, - MLSyncFileContext, -} from "services/ml/types"; +import { DetectedFace, MLSyncFileContext } from "services/ml/types"; import { EnteFile } from "types/file"; import { getRenderableImage } from "utils/file"; import { clamp } from "utils/image"; import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; class ReaderService { - async getImageBitmap( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { + async getImageBitmap(fileContext: MLSyncFileContext) { try { if (fileContext.imageBitmap) { return fileContext.imageBitmap; diff --git a/web/apps/photos/src/services/ml/types.ts b/web/apps/photos/src/services/ml/types.ts index 422cf9d4aa..f7382d3da7 100644 --- a/web/apps/photos/src/services/ml/types.ts +++ b/web/apps/photos/src/services/ml/types.ts @@ -208,7 +208,6 @@ export interface MLSearchConfig { export interface MLSyncContext { token: string; userID: number; - config: MLSyncConfig; shouldUpdateMLVersion: boolean; faceDetectionService: FaceDetectionService; From 7d122f825cb007413f8c802580617f9f37b521bc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 10:47:15 +0530 Subject: [PATCH 07/47] Inline --- .../services/machineLearning/arcfaceCropService.ts | 14 +++++++------- .../src/services/machineLearning/faceService.ts | 1 - web/apps/photos/src/services/ml/types.ts | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts index 2075d6acf8..049fa52583 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts @@ -2,7 +2,6 @@ import { Box, enlargeBox } from "services/ml/geom"; import { FaceAlignment, FaceCrop, - FaceCropConfig, FaceCropMethod, FaceCropService, FaceDetection, @@ -24,10 +23,9 @@ class ArcFaceCropService implements FaceCropService { public async getFaceCrop( imageBitmap: ImageBitmap, faceDetection: FaceDetection, - config: FaceCropConfig, ): Promise { const alignedFace = getArcfaceAlignment(faceDetection); - const faceCrop = getFaceCrop(imageBitmap, alignedFace, config); + const faceCrop = getFaceCrop(imageBitmap, alignedFace); return faceCrop; } @@ -38,19 +36,21 @@ export default new ArcFaceCropService(); export function getFaceCrop( imageBitmap: ImageBitmap, alignment: FaceAlignment, - config: FaceCropConfig, ): FaceCrop { + const padding = 0.25; + const maxSize = 256; + const alignmentBox = new Box({ x: alignment.center.x - alignment.size / 2, y: alignment.center.y - alignment.size / 2, width: alignment.size, height: alignment.size, }).round(); - const scaleForPadding = 1 + config.padding * 2; + const scaleForPadding = 1 + padding * 2; const paddedBox = enlargeBox(alignmentBox, scaleForPadding).round(); const faceImageBitmap = cropWithRotation(imageBitmap, paddedBox, 0, { - width: config.maxSize, - height: config.maxSize, + width: maxSize, + height: maxSize, }); return { diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 93e2c0a0a5..b6a7da0978 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -218,7 +218,6 @@ class FaceService { const faceCrop = await syncContext.faceCropService.getFaceCrop( imageBitmap, face.detection, - DEFAULT_ML_SYNC_CONFIG.faceCrop, ); const blobOptions = DEFAULT_ML_SYNC_CONFIG.faceCrop.blobOptions; diff --git a/web/apps/photos/src/services/ml/types.ts b/web/apps/photos/src/services/ml/types.ts index f7382d3da7..a5e97098cb 100644 --- a/web/apps/photos/src/services/ml/types.ts +++ b/web/apps/photos/src/services/ml/types.ts @@ -271,7 +271,6 @@ export interface FaceCropService { getFaceCrop( imageBitmap: ImageBitmap, face: FaceDetection, - config: FaceCropConfig, ): Promise; } From 6ce956c5bb103a90e0b7ba06b9919fc945a353ba Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 10:52:00 +0530 Subject: [PATCH 08/47] Inline --- .../photos/src/services/machineLearning/faceService.ts | 3 +-- web/apps/photos/src/utils/image/index.ts | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index b6a7da0978..6318650f6f 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -220,8 +220,7 @@ class FaceService { face.detection, ); - const blobOptions = DEFAULT_ML_SYNC_CONFIG.faceCrop.blobOptions; - const blob = await imageBitmapToBlob(faceCrop.image, blobOptions); + const blob = await imageBitmapToBlob(faceCrop.image); const cache = await openCache("face-crops"); await cache.put(face.id, blob); diff --git a/web/apps/photos/src/utils/image/index.ts b/web/apps/photos/src/utils/image/index.ts index bdaf64d735..b6f1d22aad 100644 --- a/web/apps/photos/src/utils/image/index.ts +++ b/web/apps/photos/src/utils/image/index.ts @@ -450,17 +450,17 @@ export interface BlobOptions { quality?: number; } -export async function imageBitmapToBlob( - imageBitmap: ImageBitmap, - options?: BlobOptions, -) { +export async function imageBitmapToBlob(imageBitmap: ImageBitmap) { const offscreen = new OffscreenCanvas( imageBitmap.width, imageBitmap.height, ); offscreen.getContext("2d").drawImage(imageBitmap, 0, 0); - return offscreen.convertToBlob(options); + return offscreen.convertToBlob({ + type: "image/jpeg", + quality: 0.8, + }); } export async function imageBitmapFromBlob(blob: Blob) { From 1856e344db3d9febb7bd406cc904133c0d67357e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:03:03 +0530 Subject: [PATCH 09/47] Remove unused clustering method --- web/apps/photos/package.json | 1 - .../src/services/machineLearning/machineLearningService.ts | 4 ---- web/docs/dependencies.md | 3 +++ web/yarn.lock | 5 ----- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index ac658c0ea5..a200c8ef74 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -16,7 +16,6 @@ "chrono-node": "^2.2.6", "date-fns": "^2", "debounce": "^2.0.0", - "density-clustering": "^1.3.0", "eventemitter3": "^4.0.7", "exifr": "^7.1.3", "fast-srp-hap": "^2.0.4", diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 2ebd14efeb..94fae01d2a 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -40,7 +40,6 @@ import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import arcfaceAlignmentService from "./arcfaceAlignmentService"; import arcfaceCropService from "./arcfaceCropService"; -import dbscanClusteringService from "./dbscanClusteringService"; import FaceService from "./faceService"; import hdbscanClusteringService from "./hdbscanClusteringService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; @@ -175,9 +174,6 @@ export class MLFactory { if (method === "Hdbscan") { return hdbscanClusteringService; } - if (method === "Dbscan") { - return dbscanClusteringService; - } throw Error("Unknon clustering method: " + method); } diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index 062ffeb72c..bd14b25b68 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -174,3 +174,6 @@ some cases. - [sanitize-filename](https://github.com/parshap/node-sanitize-filename) is for converting arbitrary strings into strings that are suitable for being used as filenames. + +- [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face + clustering. diff --git a/web/yarn.lock b/web/yarn.lock index 12c2d5d2b4..a18a0a0dc8 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -1896,11 +1896,6 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -density-clustering@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/density-clustering/-/density-clustering-1.3.0.tgz#dc9f59c8f0ab97e1624ac64930fd3194817dcac5" - integrity sha512-icpmBubVTwLnsaor9qH/4tG5+7+f61VcqMN3V3pm9sxxSCt2Jcs0zWOgwZW9ARJYaKD3FumIgHiMOcIMRRAzFQ== - dequal@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be" From 2c0e8c76c327f75b7ea72f060581e1a0495fc7f9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:09:02 +0530 Subject: [PATCH 10/47] From discussion --- web/apps/photos/src/services/embeddingService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 0c254800ab..17ea5a396d 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -102,6 +102,10 @@ export const syncCLIPEmbeddings = async () => { if (!response.diff?.length) { return; } + // Note: in rare cases we might get a diff entry for an embedding + // corresponding to a file which has been deleted (but whose + // embedding is enqueued for deletion). Client should expect such a + // scenario (all it has to do is just ignore them). const newEmbeddings = await Promise.all( response.diff.map(async (embedding) => { try { From cfbd7806c874b487f2e08ae166a500b8b1b598f9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:10:41 +0530 Subject: [PATCH 11/47] Inline --- .../photos/src/services/machineLearning/faceService.ts | 7 ++----- .../machineLearning/hdbscanClusteringService.ts | 10 +++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 6318650f6f..a8957d4b22 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -245,12 +245,9 @@ class FaceService { ) { // await this.init(); - const clusteringConfig = DEFAULT_ML_SYNC_CONFIG.faceClustering; - - if (!allFaces || allFaces.length < clusteringConfig.minInputSize) { + if (!allFaces || allFaces.length < 50) { log.info( - "[MLService] Too few faces to cluster, not running clustering: ", - allFaces.length, + `Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`, ); return; } diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts index 0671b0bde1..bf39cd5303 100644 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts @@ -26,11 +26,11 @@ class HdbscanClusteringService implements ClusteringService { const hdbscan = new Hdbscan({ input, - minClusterSize: config.minClusterSize, - minSamples: config.minSamples, - clusterSelectionEpsilon: config.clusterSelectionEpsilon, - clusterSelectionMethod: config.clusterSelectionMethod, - debug: config.generateDebugInfo, + minClusterSize: 3, + minSamples: 5, + clusterSelectionEpsilon: 0.6, + clusterSelectionMethod: "leaf", + debug: true, }); return { From dfd91beaffafa001d6788fba84772a46b29941dc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:13:34 +0530 Subject: [PATCH 12/47] Inline --- .../photos/src/services/machineLearning/readerService.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index 98052edfa4..9d957d70ef 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -8,7 +8,6 @@ import { DetectedFace, MLSyncFileContext } from "services/ml/types"; import { EnteFile } from "types/file"; import { getRenderableImage } from "utils/file"; import { clamp } from "utils/image"; -import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; class ReaderService { async getImageBitmap(fileContext: MLSyncFileContext) { @@ -29,7 +28,6 @@ class ReaderService { fileContext.localFile, ); } else if ( - DEFAULT_ML_SYNC_CONFIG.imageSource === "Original" && [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes( fileContext.enteFile.metadata.fileType, ) @@ -38,13 +36,14 @@ class ReaderService { fileContext.enteFile, ); } else { + // TODO-ML(MR): We don't do it on videos, when will we ever come + // here? fileContext.imageBitmap = await getThumbnailImageBitmap( fileContext.enteFile, ); } - fileContext.newMlFile.imageSource = - DEFAULT_ML_SYNC_CONFIG.imageSource; + fileContext.newMlFile.imageSource = "Original"; const { width, height } = fileContext.imageBitmap; fileContext.newMlFile.imageDimensions = { width, height }; From 0993d81b5788d4bc01db9baacaa3e31c095df0a3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:15:10 +0530 Subject: [PATCH 13/47] Remove unused clustering methods --- .../machineLearning/clusteringService.ts | 88 ------------------- .../dbscanClusteringService.ts | 37 -------- .../services/machineLearning/faceService.ts | 2 - .../hdbscanClusteringService.ts | 6 +- web/apps/photos/src/services/ml/types.ts | 20 +---- 5 files changed, 3 insertions(+), 150 deletions(-) delete mode 100644 web/apps/photos/src/services/machineLearning/clusteringService.ts delete mode 100644 web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts diff --git a/web/apps/photos/src/services/machineLearning/clusteringService.ts b/web/apps/photos/src/services/machineLearning/clusteringService.ts deleted file mode 100644 index 32c25f698c..0000000000 --- a/web/apps/photos/src/services/machineLearning/clusteringService.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { DBSCAN, KMEANS, OPTICS } from "density-clustering"; -import { Hdbscan } from "hdbscan"; -import { HdbscanInput } from "hdbscan/dist/types"; -import { - ClusteringConfig, - ClusteringInput, - ClusteringMethod, - ClusteringResults, - HdbscanResults, - Versioned, -} from "services/ml/types"; - -class ClusteringService { - private dbscan: DBSCAN; - private optics: OPTICS; - private kmeans: KMEANS; - - constructor() { - this.dbscan = new DBSCAN(); - this.optics = new OPTICS(); - this.kmeans = new KMEANS(); - } - - public clusterUsingDBSCAN( - dataset: Array>, - epsilon: number = 1.0, - minPts: number = 2, - ): ClusteringResults { - // log.info("distanceFunction", DBSCAN._); - const clusters = this.dbscan.run(dataset, epsilon, minPts); - const noise = this.dbscan.noise; - return { clusters, noise }; - } - - public clusterUsingOPTICS( - dataset: Array>, - epsilon: number = 1.0, - minPts: number = 2, - ) { - const clusters = this.optics.run(dataset, epsilon, minPts); - return { clusters, noise: [] }; - } - - public clusterUsingKMEANS( - dataset: Array>, - numClusters: number = 5, - ) { - const clusters = this.kmeans.run(dataset, numClusters); - return { clusters, noise: [] }; - } - - public clusterUsingHdbscan(hdbscanInput: HdbscanInput): HdbscanResults { - if (hdbscanInput.input.length < 10) { - throw Error("too few samples to run Hdbscan"); - } - - const hdbscan = new Hdbscan(hdbscanInput); - const clusters = hdbscan.getClusters(); - const noise = hdbscan.getNoise(); - const debugInfo = hdbscan.getDebugInfo(); - - return { clusters, noise, debugInfo }; - } - - public cluster( - method: Versioned, - input: ClusteringInput, - config: ClusteringConfig, - ) { - if (method.value === "Hdbscan") { - return this.clusterUsingHdbscan({ - input, - minClusterSize: config.minClusterSize, - debug: config.generateDebugInfo, - }); - } else if (method.value === "Dbscan") { - return this.clusterUsingDBSCAN( - input, - config.maxDistanceInsideCluster, - config.minClusterSize, - ); - } else { - throw Error("Unknown clustering method: " + method.value); - } - } -} - -export default ClusteringService; diff --git a/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts deleted file mode 100644 index 57d181de4f..0000000000 --- a/web/apps/photos/src/services/machineLearning/dbscanClusteringService.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { DBSCAN } from "density-clustering"; -import { - ClusteringConfig, - ClusteringInput, - ClusteringMethod, - ClusteringService, - HdbscanResults, - Versioned, -} from "services/ml/types"; - -class DbscanClusteringService implements ClusteringService { - public method: Versioned; - - constructor() { - this.method = { - value: "Dbscan", - version: 1, - }; - } - - public async cluster( - input: ClusteringInput, - config: ClusteringConfig, - ): Promise { - // log.info('Clustering input: ', input); - const dbscan = new DBSCAN(); - const clusters = dbscan.run( - input, - config.clusterSelectionEpsilon, - config.minClusterSize, - ); - const noise = dbscan.noise; - return { clusters, noise }; - } -} - -export default new DbscanClusteringService(); diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index a8957d4b22..c7802a08ce 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -10,7 +10,6 @@ import { type Versioned, } from "services/ml/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; -import { DEFAULT_ML_SYNC_CONFIG } from "./machineLearningService"; import ReaderService, { fetchImageBitmap, getFaceId, @@ -256,7 +255,6 @@ class FaceService { syncContext.mlLibraryData.faceClusteringResults = await syncContext.faceClusteringService.cluster( allFaces.map((f) => Array.from(f.embedding)), - DEFAULT_ML_SYNC_CONFIG.faceClustering, ); syncContext.mlLibraryData.faceClusteringMethod = syncContext.faceClusteringService.method; diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts index bf39cd5303..f6ea090d8a 100644 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts @@ -1,6 +1,5 @@ import { Hdbscan } from "hdbscan"; import { - ClusteringConfig, ClusteringInput, ClusteringMethod, ClusteringService, @@ -18,10 +17,7 @@ class HdbscanClusteringService implements ClusteringService { }; } - public async cluster( - input: ClusteringInput, - config: ClusteringConfig, - ): Promise { + public async cluster(input: ClusteringInput): Promise { // log.info('Clustering input: ', input); const hdbscan = new Hdbscan({ input, diff --git a/web/apps/photos/src/services/ml/types.ts b/web/apps/photos/src/services/ml/types.ts index a5e97098cb..e6360dc8b3 100644 --- a/web/apps/photos/src/services/ml/types.ts +++ b/web/apps/photos/src/services/ml/types.ts @@ -175,8 +175,6 @@ export interface FaceEmbeddingConfig { generateTsne?: boolean; } -export interface FaceClusteringConfig extends ClusteringConfig {} - export declare type TSNEMetric = "euclidean" | "manhattan"; export interface TSNEConfig { @@ -197,7 +195,7 @@ export interface MLSyncConfig { faceAlignment: FaceAlignmentConfig; blurDetection: BlurDetectionConfig; faceEmbedding: FaceEmbeddingConfig; - faceClustering: FaceClusteringConfig; + faceClustering: any; mlVersion: number; } @@ -294,21 +292,7 @@ export interface BlurDetectionService { export interface ClusteringService { method: Versioned; - cluster( - input: ClusteringInput, - config: ClusteringConfig, - ): Promise; -} - -export interface ClusteringConfig { - method: ClusteringMethod; - minClusterSize: number; - minSamples?: number; - clusterSelectionEpsilon?: number; - clusterSelectionMethod?: "eom" | "leaf"; - maxDistanceInsideCluster?: number; - minInputSize?: number; - generateDebugInfo?: boolean; + cluster(input: ClusteringInput): Promise; } export declare type ClusteringInput = Array>; From c127b7fc7ead00c9889273080d1da75355ed4d0e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:23:48 +0530 Subject: [PATCH 14/47] doc --- web/docs/dependencies.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index bd14b25b68..f5082b9f34 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -175,5 +175,14 @@ some cases. for converting arbitrary strings into strings that are suitable for being used as filenames. +## Face search + +- [matrix](https://github.com/mljs/matrix) and + [similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js) + are used during face alignment. + +- [transformation-matrix](https://github.com/chrvadala/transformation-matrix) + is used during face detection. + - [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face clustering. From 24c33fceb74e1f2cf826236768e928b6feb0ed20 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:28:55 +0530 Subject: [PATCH 15/47] Remove class --- .../services/machineLearning/faceService.ts | 9 ++- .../machineLearning/machineLearningService.ts | 5 +- .../services/machineLearning/readerService.ts | 78 ++++++++----------- 3 files changed, 41 insertions(+), 51 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index c7802a08ce..1f37ef536f 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -10,8 +10,9 @@ import { type Versioned, } from "services/ml/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; -import ReaderService, { +import { fetchImageBitmap, + fetchImageBitmapForContext, getFaceId, getLocalFile, } from "./readerService"; @@ -43,7 +44,7 @@ class FaceService { newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method; fileContext.newDetection = true; - const imageBitmap = await ReaderService.getImageBitmap(fileContext); + const imageBitmap = await fetchImageBitmapForContext(fileContext); const timerId = `faceDetection-${fileContext.enteFile.id}`; console.time(timerId); const faceDetections = @@ -89,7 +90,7 @@ class FaceService { return; } - const imageBitmap = await ReaderService.getImageBitmap(fileContext); + const imageBitmap = await fetchImageBitmapForContext(fileContext); newMlFile.faceCropMethod = syncContext.faceCropService.method; for (const face of newMlFile.faces) { @@ -121,7 +122,7 @@ class FaceService { fileContext.newAlignment = true; const imageBitmap = fileContext.imageBitmap || - (await ReaderService.getImageBitmap(fileContext)); + (await fetchImageBitmapForContext(fileContext)); // Execute the face alignment calculations for (const face of newMlFile.faces) { diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 94fae01d2a..27fa669f68 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -45,7 +45,8 @@ import hdbscanClusteringService from "./hdbscanClusteringService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import PeopleService from "./peopleService"; -import ReaderService from "./readerService"; + +import { fetchImageBitmapForContext } from "./readerService"; import yoloFaceDetectionService from "./yoloFaceDetectionService"; /** @@ -581,7 +582,7 @@ class MachineLearningService { } try { - await ReaderService.getImageBitmap(fileContext); + await fetchImageBitmapForContext(fileContext); await Promise.all([ this.syncFileAnalyzeFaces(syncContext, fileContext), ]); diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index 9d957d70ef..3e82a9dac7 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -9,52 +9,40 @@ import { EnteFile } from "types/file"; import { getRenderableImage } from "utils/file"; import { clamp } from "utils/image"; -class ReaderService { - async getImageBitmap(fileContext: MLSyncFileContext) { - try { - if (fileContext.imageBitmap) { - return fileContext.imageBitmap; - } - if (fileContext.localFile) { - if ( - fileContext.enteFile.metadata.fileType !== FILE_TYPE.IMAGE - ) { - throw new Error( - "Local file of only image type is supported", - ); - } - fileContext.imageBitmap = await getLocalFileImageBitmap( - fileContext.enteFile, - fileContext.localFile, - ); - } else if ( - [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes( - fileContext.enteFile.metadata.fileType, - ) - ) { - fileContext.imageBitmap = await fetchImageBitmap( - fileContext.enteFile, - ); - } else { - // TODO-ML(MR): We don't do it on videos, when will we ever come - // here? - fileContext.imageBitmap = await getThumbnailImageBitmap( - fileContext.enteFile, - ); - } - - fileContext.newMlFile.imageSource = "Original"; - const { width, height } = fileContext.imageBitmap; - fileContext.newMlFile.imageDimensions = { width, height }; - - return fileContext.imageBitmap; - } catch (e) { - log.error("failed to create image bitmap", e); - throw e; - } +export const fetchImageBitmapForContext = async ( + fileContext: MLSyncFileContext, +) => { + if (fileContext.imageBitmap) { + return fileContext.imageBitmap; } -} -export default new ReaderService(); + if (fileContext.localFile) { + if (fileContext.enteFile.metadata.fileType !== FILE_TYPE.IMAGE) { + throw new Error("Local file of only image type is supported"); + } + fileContext.imageBitmap = await getLocalFileImageBitmap( + fileContext.enteFile, + fileContext.localFile, + ); + } else if ( + [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes( + fileContext.enteFile.metadata.fileType, + ) + ) { + fileContext.imageBitmap = await fetchImageBitmap(fileContext.enteFile); + } else { + // TODO-ML(MR): We don't do it on videos, when will we ever come + // here? + fileContext.imageBitmap = await getThumbnailImageBitmap( + fileContext.enteFile, + ); + } + + fileContext.newMlFile.imageSource = "Original"; + const { width, height } = fileContext.imageBitmap; + fileContext.newMlFile.imageDimensions = { width, height }; + + return fileContext.imageBitmap; +}; export async function getLocalFile(fileId: number) { const localFiles = await getLocalFiles(); From 19f06e6494e2c09dd19f4c5f981541ecafae026e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:33:03 +0530 Subject: [PATCH 16/47] Rename --- .../Search/SearchBar/searchInput/MenuWithPeople.tsx | 2 +- .../src/components/Search/SearchBar/searchInput/index.tsx | 2 +- web/apps/photos/src/components/ml/PeopleList.tsx | 4 ++-- web/apps/photos/src/services/{ml => face}/db.ts | 2 +- web/apps/photos/src/services/{ml => face}/face.worker.ts | 2 +- web/apps/photos/src/services/{ml => face}/geom.ts | 0 .../{machineLearning/readerService.ts => face/image.ts} | 4 ++-- .../photos/src/services/{ml/face.ts => face/index.ts} | 2 +- web/apps/photos/src/services/{ml => face}/types.ts | 2 +- .../services/machineLearning/arcfaceAlignmentService.ts | 4 ++-- .../src/services/machineLearning/arcfaceCropService.ts | 4 ++-- .../photos/src/services/machineLearning/faceService.ts | 6 +++--- .../services/machineLearning/hdbscanClusteringService.ts | 2 +- .../machineLearning/laplacianBlurDetectionService.ts | 2 +- .../services/machineLearning/machineLearningService.ts | 8 ++++---- .../photos/src/services/machineLearning/mlWorkManager.ts | 8 ++++---- .../machineLearning/mobileFaceNetEmbeddingService.ts | 2 +- .../photos/src/services/machineLearning/peopleService.ts | 6 +++--- .../services/machineLearning/yoloFaceDetectionService.ts | 4 ++-- web/apps/photos/src/services/searchService.ts | 4 ++-- web/apps/photos/src/types/search/index.ts | 4 ++-- web/apps/photos/src/utils/image/index.ts | 4 ++-- 22 files changed, 39 insertions(+), 39 deletions(-) rename web/apps/photos/src/services/{ml => face}/db.ts (99%) rename web/apps/photos/src/services/{ml => face}/face.worker.ts (94%) rename web/apps/photos/src/services/{ml => face}/geom.ts (100%) rename web/apps/photos/src/services/{machineLearning/readerService.ts => face/image.ts} (96%) rename web/apps/photos/src/services/{ml/face.ts => face/index.ts} (81%) rename web/apps/photos/src/services/{ml => face}/types.ts (99%) diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index b9b7ea88d5..3b739520e2 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -5,7 +5,7 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext } from "react"; import { components } from "react-select"; -import { IndexStatus } from "services/ml/db"; +import { IndexStatus } from "services/face/db"; import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index da462a3b5a..1e62422dc6 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -9,8 +9,8 @@ import { useCallback, useContext, useEffect, useRef, useState } from "react"; import { components } from "react-select"; import AsyncSelect from "react-select/async"; import { InputActionMeta } from "react-select/src/types"; +import { Person } from "services/face/types"; import { City } from "services/locationSearchService"; -import { Person } from "services/ml/types"; import { getAutoCompleteSuggestions, getDefaultOptions, diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 4ff61e044d..534df6499a 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -3,8 +3,8 @@ import { Skeleton, styled } from "@mui/material"; import { Legend } from "components/PhotoViewer/styledComponents/Legend"; import { t } from "i18next"; import React, { useEffect, useState } from "react"; -import mlIDbStorage from "services/ml/db"; -import { Face, Person, type MlFileData } from "services/ml/types"; +import mlIDbStorage from "services/face/db"; +import { Face, Person, type MlFileData } from "services/face/types"; import { EnteFile } from "types/file"; const FaceChipContainer = styled("div")` diff --git a/web/apps/photos/src/services/ml/db.ts b/web/apps/photos/src/services/face/db.ts similarity index 99% rename from web/apps/photos/src/services/ml/db.ts rename to web/apps/photos/src/services/face/db.ts index 7723ebeae5..399bfff1a8 100644 --- a/web/apps/photos/src/services/ml/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -9,11 +9,11 @@ import { openDB, } from "idb"; import isElectron from "is-electron"; +import { Face, MLLibraryData, MlFileData, Person } from "services/face/types"; import { DEFAULT_ML_SEARCH_CONFIG, MAX_ML_SYNC_ERROR_COUNT, } from "services/machineLearning/machineLearningService"; -import { Face, MLLibraryData, MlFileData, Person } from "services/ml/types"; export interface IndexStatus { outOfSyncFilesExists: boolean; diff --git a/web/apps/photos/src/services/ml/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts similarity index 94% rename from web/apps/photos/src/services/ml/face.worker.ts rename to web/apps/photos/src/services/face/face.worker.ts index aa650d4aff..8c9d094987 100644 --- a/web/apps/photos/src/services/ml/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -1,7 +1,7 @@ import log from "@/next/log"; import { expose } from "comlink"; +import { MachineLearningWorker } from "services/face/types"; import mlService from "services/machineLearning/machineLearningService"; -import { MachineLearningWorker } from "services/ml/types"; import { EnteFile } from "types/file"; export class DedicatedMLWorker implements MachineLearningWorker { diff --git a/web/apps/photos/src/services/ml/geom.ts b/web/apps/photos/src/services/face/geom.ts similarity index 100% rename from web/apps/photos/src/services/ml/geom.ts rename to web/apps/photos/src/services/face/geom.ts diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/face/image.ts similarity index 96% rename from web/apps/photos/src/services/machineLearning/readerService.ts rename to web/apps/photos/src/services/face/image.ts index 3e82a9dac7..1ddcc70f66 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/face/image.ts @@ -2,9 +2,9 @@ import { FILE_TYPE } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import DownloadManager from "services/download"; +import { Dimensions } from "services/face/geom"; +import { DetectedFace, MLSyncFileContext } from "services/face/types"; import { getLocalFiles } from "services/fileService"; -import { Dimensions } from "services/ml/geom"; -import { DetectedFace, MLSyncFileContext } from "services/ml/types"; import { EnteFile } from "types/file"; import { getRenderableImage } from "utils/file"; import { clamp } from "utils/image"; diff --git a/web/apps/photos/src/services/ml/face.ts b/web/apps/photos/src/services/face/index.ts similarity index 81% rename from web/apps/photos/src/services/ml/face.ts rename to web/apps/photos/src/services/face/index.ts index 7eb6980e0f..86fa9ab20b 100644 --- a/web/apps/photos/src/services/ml/face.ts +++ b/web/apps/photos/src/services/face/index.ts @@ -1,5 +1,5 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import type { DedicatedMLWorker } from "services/ml/face.worker"; +import type { DedicatedMLWorker } from "services/face/face.worker"; const createFaceWebWorker = () => new Worker(new URL("face.worker.ts", import.meta.url)); diff --git a/web/apps/photos/src/services/ml/types.ts b/web/apps/photos/src/services/face/types.ts similarity index 99% rename from web/apps/photos/src/services/ml/types.ts rename to web/apps/photos/src/services/face/types.ts index e6360dc8b3..1a5e75a1b3 100644 --- a/web/apps/photos/src/services/ml/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -1,6 +1,6 @@ import { DebugInfo } from "hdbscan"; import PQueue from "p-queue"; -import { Dimensions } from "services/ml/geom"; +import { Dimensions } from "services/face/geom"; import { EnteFile } from "types/file"; import { Box, Point } from "./geom"; diff --git a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts b/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts index f23a065c83..749da95919 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts @@ -1,12 +1,12 @@ import { Matrix } from "ml-matrix"; -import { Point } from "services/ml/geom"; +import { Point } from "services/face/geom"; import { FaceAlignment, FaceAlignmentMethod, FaceAlignmentService, FaceDetection, Versioned, -} from "services/ml/types"; +} from "services/face/types"; import { getSimilarityTransformation } from "similarity-transformation"; class ArcfaceAlignmentService implements FaceAlignmentService { diff --git a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts index 049fa52583..9aff4b606d 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts @@ -1,4 +1,4 @@ -import { Box, enlargeBox } from "services/ml/geom"; +import { Box, enlargeBox } from "services/face/geom"; import { FaceAlignment, FaceCrop, @@ -6,7 +6,7 @@ import { FaceCropService, FaceDetection, Versioned, -} from "services/ml/types"; +} from "services/face/types"; import { cropWithRotation } from "utils/image"; import { getArcfaceAlignment } from "./arcfaceAlignmentService"; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 1f37ef536f..99c6bd99e1 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -1,6 +1,6 @@ import { openCache } from "@/next/blob-cache"; import log from "@/next/log"; -import mlIDbStorage from "services/ml/db"; +import mlIDbStorage from "services/face/db"; import { DetectedFace, Face, @@ -8,14 +8,14 @@ import { MLSyncFileContext, type FaceAlignment, type Versioned, -} from "services/ml/types"; +} from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { fetchImageBitmap, fetchImageBitmapForContext, getFaceId, getLocalFile, -} from "./readerService"; +} from "../face/image"; class FaceService { async syncFileFaceDetections( diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts index f6ea090d8a..1f508d5450 100644 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts @@ -5,7 +5,7 @@ import { ClusteringService, HdbscanResults, Versioned, -} from "services/ml/types"; +} from "services/face/types"; class HdbscanClusteringService implements ClusteringService { public method: Versioned; diff --git a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts index f1d7bf5008..a4d8000cf0 100644 --- a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts @@ -3,7 +3,7 @@ import { BlurDetectionService, Face, Versioned, -} from "services/ml/types"; +} from "services/face/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; import { mobileFaceNetFaceSize } from "./mobileFaceNetEmbeddingService"; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 27fa669f68..efc470c4d4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -10,8 +10,7 @@ import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; import downloadManager from "services/download"; import { putEmbedding } from "services/embeddingService"; -import { getLocalFiles } from "services/fileService"; -import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/ml/db"; +import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { BlurDetectionMethod, BlurDetectionService, @@ -35,7 +34,8 @@ import { MLSyncFileContext, MLSyncResult, MlFileData, -} from "services/ml/types"; +} from "services/face/types"; +import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import arcfaceAlignmentService from "./arcfaceAlignmentService"; @@ -46,7 +46,7 @@ import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import PeopleService from "./peopleService"; -import { fetchImageBitmapForContext } from "./readerService"; +import { fetchImageBitmapForContext } from "../face/image"; import yoloFaceDetectionService from "./yoloFaceDetectionService"; /** diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 6ce5d80084..52fb43f9cf 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -5,10 +5,10 @@ import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; -import mlIDbStorage from "services/ml/db"; -import { createFaceComlinkWorker } from "services/ml/face"; -import type { DedicatedMLWorker } from "services/ml/face.worker"; -import { MLSyncResult } from "services/ml/types"; +import mlIDbStorage from "services/face/db"; +import { createFaceComlinkWorker } from "services/face"; +import type { DedicatedMLWorker } from "services/face/face.worker"; +import { MLSyncResult } from "services/face/types"; import { EnteFile } from "types/file"; import { logQueueStats } from "./machineLearningService"; diff --git a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts b/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts index 1b2205801e..46288d1699 100644 --- a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts +++ b/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts @@ -4,7 +4,7 @@ import { FaceEmbeddingMethod, FaceEmbeddingService, Versioned, -} from "services/ml/types"; +} from "services/face/types"; export const mobileFaceNetFaceSize = 112; diff --git a/web/apps/photos/src/services/machineLearning/peopleService.ts b/web/apps/photos/src/services/machineLearning/peopleService.ts index b26153f622..c485a62be0 100644 --- a/web/apps/photos/src/services/machineLearning/peopleService.ts +++ b/web/apps/photos/src/services/machineLearning/peopleService.ts @@ -1,8 +1,8 @@ import log from "@/next/log"; -import mlIDbStorage from "services/ml/db"; -import { Face, MLSyncContext, Person } from "services/ml/types"; +import mlIDbStorage from "services/face/db"; +import { Face, MLSyncContext, Person } from "services/face/types"; +import { fetchImageBitmap, getLocalFile } from "../face/image"; import FaceService, { isDifferentOrOld } from "./faceService"; -import { fetchImageBitmap, getLocalFile } from "./readerService"; class PeopleService { async syncPeopleIndex(syncContext: MLSyncContext) { diff --git a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts index 8856576bee..e9e4fda622 100644 --- a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts @@ -6,13 +6,13 @@ import { Point, boxFromBoundingBox, newBox, -} from "services/ml/geom"; +} from "services/face/geom"; import { FaceDetection, FaceDetectionMethod, FaceDetectionService, Versioned, -} from "services/ml/types"; +} from "services/face/types"; import { Matrix, applyToPoint, diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 54c69ee080..d646ecd007 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,9 +2,9 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; +import mlIDbStorage from "services/face/db"; +import { Person } from "services/face/types"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; -import mlIDbStorage from "services/ml/db"; -import { Person } from "services/ml/types"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index e08b842a30..aa5f128042 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,7 +1,7 @@ import { FILE_TYPE } from "@/media/file-type"; +import { IndexStatus } from "services/face/db"; +import { Person } from "services/face/types"; import { City } from "services/locationSearchService"; -import { IndexStatus } from "services/ml/db"; -import { Person } from "services/ml/types"; import { LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/utils/image/index.ts b/web/apps/photos/src/utils/image/index.ts index b6f1d22aad..7583f97c27 100644 --- a/web/apps/photos/src/utils/image/index.ts +++ b/web/apps/photos/src/utils/image/index.ts @@ -1,8 +1,8 @@ // these utils only work in env where OffscreenCanvas is available import { Matrix, inverse } from "ml-matrix"; -import { Box, Dimensions, enlargeBox } from "services/ml/geom"; -import { FaceAlignment } from "services/ml/types"; +import { Box, Dimensions, enlargeBox } from "services/face/geom"; +import { FaceAlignment } from "services/face/types"; export function normalizePixelBetween0And1(pixelValue: number) { return pixelValue / 255.0; From fbebbd3583a45dd4ac9c5d91d36e604b6f34a54e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 11:43:04 +0530 Subject: [PATCH 17/47] Clean alignment --- .../align.ts} | 38 ++++--------------- web/apps/photos/src/services/face/types.ts | 6 --- .../machineLearning/arcfaceCropService.ts | 6 +-- .../services/machineLearning/faceService.ts | 20 ++++++---- .../machineLearning/machineLearningService.ts | 16 -------- 5 files changed, 23 insertions(+), 63 deletions(-) rename web/apps/photos/src/services/{machineLearning/arcfaceAlignmentService.ts => face/align.ts} (73%) diff --git a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts b/web/apps/photos/src/services/face/align.ts similarity index 73% rename from web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts rename to web/apps/photos/src/services/face/align.ts index 749da95919..7a3bf7a044 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceAlignmentService.ts +++ b/web/apps/photos/src/services/face/align.ts @@ -1,31 +1,8 @@ import { Matrix } from "ml-matrix"; import { Point } from "services/face/geom"; -import { - FaceAlignment, - FaceAlignmentMethod, - FaceAlignmentService, - FaceDetection, - Versioned, -} from "services/face/types"; +import { FaceAlignment, FaceDetection } from "services/face/types"; import { getSimilarityTransformation } from "similarity-transformation"; -class ArcfaceAlignmentService implements FaceAlignmentService { - public method: Versioned; - - constructor() { - this.method = { - value: "ArcFace", - version: 1, - }; - } - - public getFaceAlignment(faceDetection: FaceDetection): FaceAlignment { - return getArcfaceAlignment(faceDetection); - } -} - -export default new ArcfaceAlignmentService(); - const ARCFACE_LANDMARKS = [ [38.2946, 51.6963], [73.5318, 51.5014], @@ -43,9 +20,12 @@ const ARC_FACE_5_LANDMARKS = [ [70.7299, 92.2041], ] as Array<[number, number]>; -export function getArcfaceAlignment( - faceDetection: FaceDetection, -): FaceAlignment { +/** + * Compute and return an {@link FaceAlignment} for the given face detection. + * + * @param faceDetection A geometry indicating a face detected in an image. + */ +export const faceAlignment = (faceDetection: FaceDetection): FaceAlignment => { const landmarkCount = faceDetection.landmarks.length; return getFaceAlignmentUsingSimilarityTransform( faceDetection, @@ -54,12 +34,11 @@ export function getArcfaceAlignment( ARCFACE_LANDMARKS_FACE_SIZE, ), ); -} +}; function getFaceAlignmentUsingSimilarityTransform( faceDetection: FaceDetection, alignedLandmarks: Array<[number, number]>, - // alignmentMethod: Versioned ): FaceAlignment { const landmarksMat = new Matrix( faceDetection.landmarks @@ -90,7 +69,6 @@ function getFaceAlignmentUsingSimilarityTransform( simTransform.rotation.get(0, 1), simTransform.rotation.get(0, 0), ); - // log.info({ affineMatrix, meanTranslation, centerMat, center, toMean: simTransform.toMean, fromMean: simTransform.fromMean, size }); return { affineMatrix, diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index 1a5e75a1b3..b652657ddb 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -210,7 +210,6 @@ export interface MLSyncContext { faceDetectionService: FaceDetectionService; faceCropService: FaceCropService; - faceAlignmentService: FaceAlignmentService; faceEmbeddingService: FaceEmbeddingService; blurDetectionService: BlurDetectionService; faceClusteringService: ClusteringService; @@ -272,11 +271,6 @@ export interface FaceCropService { ): Promise; } -export interface FaceAlignmentService { - method: Versioned; - getFaceAlignment(faceDetection: FaceDetection): FaceAlignment; -} - export interface FaceEmbeddingService { method: Versioned; faceSize: number; diff --git a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts index 9aff4b606d..81f2d4de5e 100644 --- a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts +++ b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts @@ -8,7 +8,7 @@ import { Versioned, } from "services/face/types"; import { cropWithRotation } from "utils/image"; -import { getArcfaceAlignment } from "./arcfaceAlignmentService"; +import { faceAlignment } from "../face/align"; class ArcFaceCropService implements FaceCropService { public method: Versioned; @@ -24,8 +24,8 @@ class ArcFaceCropService implements FaceCropService { imageBitmap: ImageBitmap, faceDetection: FaceDetection, ): Promise { - const alignedFace = getArcfaceAlignment(faceDetection); - const faceCrop = getFaceCrop(imageBitmap, alignedFace); + const alignment = faceAlignment(faceDetection); + const faceCrop = getFaceCrop(imageBitmap, alignment); return faceCrop; } diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 99c6bd99e1..7183db1f10 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -1,5 +1,6 @@ import { openCache } from "@/next/blob-cache"; import log from "@/next/log"; +import { faceAlignment } from "services/face/align"; import mlIDbStorage from "services/face/db"; import { DetectedFace, @@ -103,12 +104,14 @@ class FaceService { fileContext: MLSyncFileContext, ): Promise { const { oldMlFile, newMlFile } = fileContext; + // TODO-ML(MR): + const method = { + value: "ArcFace", + version: 1, + }; if ( !fileContext.newDetection && - !isDifferentOrOld( - oldMlFile?.faceAlignmentMethod, - syncContext.faceAlignmentService.method, - ) && + !isDifferentOrOld(oldMlFile?.faceAlignmentMethod, method) && areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) ) { for (const [index, face] of newMlFile.faces.entries()) { @@ -118,7 +121,10 @@ class FaceService { return; } - newMlFile.faceAlignmentMethod = syncContext.faceAlignmentService.method; + newMlFile.faceAlignmentMethod = { + value: "ArcFace", + version: 1, + }; fileContext.newAlignment = true; const imageBitmap = fileContext.imageBitmap || @@ -126,9 +132,7 @@ class FaceService { // Execute the face alignment calculations for (const face of newMlFile.faces) { - face.alignment = syncContext.faceAlignmentService.getFaceAlignment( - face.detection, - ); + face.alignment = faceAlignment(face.detection); } // Extract face images and convert to Float32Array const faceAlignments = newMlFile.faces.map((f) => f.alignment); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index efc470c4d4..eb3d505581 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -17,8 +17,6 @@ import { ClusteringMethod, ClusteringService, Face, - FaceAlignmentMethod, - FaceAlignmentService, FaceCropMethod, FaceCropService, FaceDetection, @@ -38,7 +36,6 @@ import { import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import arcfaceAlignmentService from "./arcfaceAlignmentService"; import arcfaceCropService from "./arcfaceCropService"; import FaceService from "./faceService"; import hdbscanClusteringService from "./hdbscanClusteringService"; @@ -139,16 +136,6 @@ export class MLFactory { throw Error("Unknon face crop method: " + method); } - public static getFaceAlignmentService( - method: FaceAlignmentMethod, - ): FaceAlignmentService { - if (method === "ArcFace") { - return arcfaceAlignmentService; - } - - throw Error("Unknon face alignment method: " + method); - } - public static getBlurDetectionService( method: BlurDetectionMethod, ): BlurDetectionService { @@ -187,7 +174,6 @@ export class LocalMLSyncContext implements MLSyncContext { public faceDetectionService: FaceDetectionService; public faceCropService: FaceCropService; - public faceAlignmentService: FaceAlignmentService; public blurDetectionService: BlurDetectionService; public faceEmbeddingService: FaceEmbeddingService; public faceClusteringService: ClusteringService; @@ -225,8 +211,6 @@ export class LocalMLSyncContext implements MLSyncContext { this.faceDetectionService = MLFactory.getFaceDetectionService("YoloFace"); this.faceCropService = MLFactory.getFaceCropService("ArcFace"); - this.faceAlignmentService = - MLFactory.getFaceAlignmentService("ArcFace"); this.blurDetectionService = MLFactory.getBlurDetectionService("Laplacian"); this.faceEmbeddingService = From 2a35b0ec9cb5903a7466d3355c2f015aeccfd296 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:04:44 +0530 Subject: [PATCH 18/47] Inline clustering --- web/apps/photos/src/services/face/cluster.ts | 34 ++++++++++++++++ web/apps/photos/src/services/face/types.ts | 22 +--------- .../services/machineLearning/faceService.ts | 14 ++++--- .../hdbscanClusteringService.ts | 40 ------------------- .../machineLearning/machineLearningService.ts | 15 ------- .../services/machineLearning/peopleService.ts | 8 ++-- 6 files changed, 48 insertions(+), 85 deletions(-) create mode 100644 web/apps/photos/src/services/face/cluster.ts delete mode 100644 web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts diff --git a/web/apps/photos/src/services/face/cluster.ts b/web/apps/photos/src/services/face/cluster.ts new file mode 100644 index 0000000000..9ddf156cc7 --- /dev/null +++ b/web/apps/photos/src/services/face/cluster.ts @@ -0,0 +1,34 @@ +import { Hdbscan, type DebugInfo } from "hdbscan"; +import { type Cluster } from "services/face/types"; + +export interface ClusterFacesResult { + clusters: Array; + noise: Cluster; + debugInfo?: DebugInfo; +} + +/** + * Cluster the given {@link faceEmbeddings}. + * + * @param faceEmbeddings An array of embeddings produced by our face indexing + * pipeline. Each embedding is for a face detected in an image (a single image + * may have multiple faces detected within it). + */ +export const clusterFaces = async ( + faceEmbeddings: Array>, +): Promise => { + const hdbscan = new Hdbscan({ + input: faceEmbeddings, + minClusterSize: 3, + minSamples: 5, + clusterSelectionEpsilon: 0.6, + clusterSelectionMethod: "leaf", + debug: true, + }); + + return { + clusters: hdbscan.getClusters(), + noise: hdbscan.getNoise(), + debugInfo: hdbscan.getDebugInfo(), + }; +}; diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index b652657ddb..f7a88d9c63 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -1,5 +1,5 @@ -import { DebugInfo } from "hdbscan"; import PQueue from "p-queue"; +import type { ClusterFacesResult } from "services/face/cluster"; import { Dimensions } from "services/face/geom"; import { EnteFile } from "types/file"; import { Box, Point } from "./geom"; @@ -17,15 +17,6 @@ export declare type FaceDescriptor = Float32Array; export declare type Cluster = Array; -export interface ClusteringResults { - clusters: Array; - noise: Cluster; -} - -export interface HdbscanResults extends ClusteringResults { - debugInfo?: DebugInfo; -} - export interface FacesCluster { faces: Cluster; summary?: FaceDescriptor; @@ -212,7 +203,6 @@ export interface MLSyncContext { faceCropService: FaceCropService; faceEmbeddingService: FaceEmbeddingService; blurDetectionService: BlurDetectionService; - faceClusteringService: ClusteringService; localFilesMap: Map; outOfSyncFiles: EnteFile[]; @@ -246,7 +236,7 @@ export interface MLSyncFileContext { export interface MLLibraryData { faceClusteringMethod?: Versioned; - faceClusteringResults?: ClusteringResults; + faceClusteringResults?: ClusterFacesResult; faceClustersWithNoise?: FacesClustersWithNoise; } @@ -283,14 +273,6 @@ export interface BlurDetectionService { detectBlur(alignedFaces: Float32Array, faces: Face[]): number[]; } -export interface ClusteringService { - method: Versioned; - - cluster(input: ClusteringInput): Promise; -} - -export declare type ClusteringInput = Array>; - export interface MachineLearningWorker { closeLocalSyncContext(): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 7183db1f10..8b521add99 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -11,6 +11,7 @@ import { type Versioned, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { clusterFaces } from "../face/cluster"; import { fetchImageBitmap, fetchImageBitmapForContext, @@ -257,12 +258,13 @@ class FaceService { } log.info("Running clustering allFaces: ", allFaces.length); - syncContext.mlLibraryData.faceClusteringResults = - await syncContext.faceClusteringService.cluster( - allFaces.map((f) => Array.from(f.embedding)), - ); - syncContext.mlLibraryData.faceClusteringMethod = - syncContext.faceClusteringService.method; + syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( + allFaces.map((f) => Array.from(f.embedding)), + ); + syncContext.mlLibraryData.faceClusteringMethod = { + value: "Hdbscan", + version: 1, + }; log.info( "[MLService] Got face clustering results: ", JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), diff --git a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts b/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts deleted file mode 100644 index 1f508d5450..0000000000 --- a/web/apps/photos/src/services/machineLearning/hdbscanClusteringService.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Hdbscan } from "hdbscan"; -import { - ClusteringInput, - ClusteringMethod, - ClusteringService, - HdbscanResults, - Versioned, -} from "services/face/types"; - -class HdbscanClusteringService implements ClusteringService { - public method: Versioned; - - constructor() { - this.method = { - value: "Hdbscan", - version: 1, - }; - } - - public async cluster(input: ClusteringInput): Promise { - // log.info('Clustering input: ', input); - const hdbscan = new Hdbscan({ - input, - - minClusterSize: 3, - minSamples: 5, - clusterSelectionEpsilon: 0.6, - clusterSelectionMethod: "leaf", - debug: true, - }); - - return { - clusters: hdbscan.getClusters(), - noise: hdbscan.getNoise(), - debugInfo: hdbscan.getDebugInfo(), - }; - } -} - -export default new HdbscanClusteringService(); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index eb3d505581..a944e4cf2e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -14,8 +14,6 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { BlurDetectionMethod, BlurDetectionService, - ClusteringMethod, - ClusteringService, Face, FaceCropMethod, FaceCropService, @@ -38,7 +36,6 @@ import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import arcfaceCropService from "./arcfaceCropService"; import FaceService from "./faceService"; -import hdbscanClusteringService from "./hdbscanClusteringService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import PeopleService from "./peopleService"; @@ -155,16 +152,6 @@ export class MLFactory { throw Error("Unknon face embedding method: " + method); } - - public static getClusteringService( - method: ClusteringMethod, - ): ClusteringService { - if (method === "Hdbscan") { - return hdbscanClusteringService; - } - - throw Error("Unknon clustering method: " + method); - } } export class LocalMLSyncContext implements MLSyncContext { @@ -176,7 +163,6 @@ export class LocalMLSyncContext implements MLSyncContext { public faceCropService: FaceCropService; public blurDetectionService: BlurDetectionService; public faceEmbeddingService: FaceEmbeddingService; - public faceClusteringService: ClusteringService; public localFilesMap: Map; public outOfSyncFiles: EnteFile[]; @@ -215,7 +201,6 @@ export class LocalMLSyncContext implements MLSyncContext { MLFactory.getBlurDetectionService("Laplacian"); this.faceEmbeddingService = MLFactory.getFaceEmbeddingService("MobileFaceNet"); - this.faceClusteringService = MLFactory.getClusteringService("Hdbscan"); this.outOfSyncFiles = []; this.nSyncedFiles = 0; diff --git a/web/apps/photos/src/services/machineLearning/peopleService.ts b/web/apps/photos/src/services/machineLearning/peopleService.ts index c485a62be0..2224f42003 100644 --- a/web/apps/photos/src/services/machineLearning/peopleService.ts +++ b/web/apps/photos/src/services/machineLearning/peopleService.ts @@ -9,10 +9,10 @@ class PeopleService { const filesVersion = await mlIDbStorage.getIndexVersion("files"); if ( filesVersion <= (await mlIDbStorage.getIndexVersion("people")) && - !isDifferentOrOld( - syncContext.mlLibraryData?.faceClusteringMethod, - syncContext.faceClusteringService.method, - ) + !isDifferentOrOld(syncContext.mlLibraryData?.faceClusteringMethod, { + value: "Hdbscan", + version: 1, + }) ) { log.info( "[MLService] Skipping people index as already synced to latest version", From c48042546e2f5b8bd0734964bf5aa2d7473a16cc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:15:11 +0530 Subject: [PATCH 19/47] Inline and move --- .../machineLearning/machineLearningService.ts | 4 +- .../services/machineLearning/peopleService.ts | 113 ------------------ 2 files changed, 2 insertions(+), 115 deletions(-) delete mode 100644 web/apps/photos/src/services/machineLearning/peopleService.ts diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index a944e4cf2e..26d9e69dd9 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -38,9 +38,9 @@ import arcfaceCropService from "./arcfaceCropService"; import FaceService from "./faceService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; -import PeopleService from "./peopleService"; import { fetchImageBitmapForContext } from "../face/image"; +import { syncPeopleIndex } from "../face/people"; import yoloFaceDetectionService from "./yoloFaceDetectionService"; /** @@ -628,7 +628,7 @@ class MachineLearningService { public async syncIndex(syncContext: MLSyncContext) { await this.getMLLibraryData(syncContext); - await PeopleService.syncPeopleIndex(syncContext); + await syncPeopleIndex(syncContext); await this.persistMLLibraryData(syncContext); } diff --git a/web/apps/photos/src/services/machineLearning/peopleService.ts b/web/apps/photos/src/services/machineLearning/peopleService.ts deleted file mode 100644 index 2224f42003..0000000000 --- a/web/apps/photos/src/services/machineLearning/peopleService.ts +++ /dev/null @@ -1,113 +0,0 @@ -import log from "@/next/log"; -import mlIDbStorage from "services/face/db"; -import { Face, MLSyncContext, Person } from "services/face/types"; -import { fetchImageBitmap, getLocalFile } from "../face/image"; -import FaceService, { isDifferentOrOld } from "./faceService"; - -class PeopleService { - async syncPeopleIndex(syncContext: MLSyncContext) { - const filesVersion = await mlIDbStorage.getIndexVersion("files"); - if ( - filesVersion <= (await mlIDbStorage.getIndexVersion("people")) && - !isDifferentOrOld(syncContext.mlLibraryData?.faceClusteringMethod, { - value: "Hdbscan", - version: 1, - }) - ) { - log.info( - "[MLService] Skipping people index as already synced to latest version", - ); - return; - } - - // TODO: have faces addresable through fileId + faceId - // to avoid index based addressing, which is prone to wrong results - // one way could be to match nearest face within threshold in the file - const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext); - const allFaces = getAllFacesFromMap(allFacesMap); - - await FaceService.runFaceClustering(syncContext, allFaces); - await this.syncPeopleFromClusters(syncContext, allFacesMap, allFaces); - - await mlIDbStorage.setIndexVersion("people", filesVersion); - } - - private async syncPeopleFromClusters( - syncContext: MLSyncContext, - allFacesMap: Map>, - allFaces: Array, - ) { - const clusters = - syncContext.mlLibraryData.faceClusteringResults?.clusters; - if (!clusters || clusters.length < 1) { - return; - } - - for (const face of allFaces) { - face.personId = undefined; - } - await mlIDbStorage.clearAllPeople(); - for (const [index, cluster] of clusters.entries()) { - const faces = cluster.map((f) => allFaces[f]).filter((f) => f); - - // TODO: take default display face from last leaves of hdbscan clusters - const personFace = findFirstIfSorted( - faces, - (a, b) => b.detection.probability - a.detection.probability, - ); - - if (personFace && !personFace.crop?.cacheKey) { - const file = await getLocalFile(personFace.fileId); - const imageBitmap = await fetchImageBitmap(file); - await FaceService.saveFaceCrop( - imageBitmap, - personFace, - syncContext, - ); - } - - const person: Person = { - id: index, - files: faces.map((f) => f.fileId), - displayFaceId: personFace?.id, - faceCropCacheKey: personFace?.crop?.cacheKey, - }; - - await mlIDbStorage.putPerson(person); - - faces.forEach((face) => { - face.personId = person.id; - }); - // log.info("Creating person: ", person, faces); - } - - await mlIDbStorage.updateFaces(allFacesMap); - } -} - -export default new PeopleService(); - -function findFirstIfSorted( - elements: Array, - comparator: (a: T, b: T) => number, -) { - if (!elements || elements.length < 1) { - return; - } - let first = elements[0]; - - for (let i = 1; i < elements.length; i++) { - const comp = comparator(elements[i], first); - if (comp < 0) { - first = elements[i]; - } - } - - return first; -} - -function getAllFacesFromMap(allFacesMap: Map>) { - const allFaces = [...allFacesMap.values()].flat(); - - return allFaces; -} From 79aea6a979b0892e2e49eb9a5fdf93df3ec9c694 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:24:00 +0530 Subject: [PATCH 20/47] Inline --- web/apps/photos/src/services/face/people.ts | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 web/apps/photos/src/services/face/people.ts diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts new file mode 100644 index 0000000000..8a0f0e969a --- /dev/null +++ b/web/apps/photos/src/services/face/people.ts @@ -0,0 +1,81 @@ +import mlIDbStorage from "services/face/db"; +import { Face, MLSyncContext, Person } from "services/face/types"; +import FaceService from "../machineLearning/faceService"; +import { fetchImageBitmap, getLocalFile } from "./image"; + +export const syncPeopleIndex = async (syncContext: MLSyncContext) => { + const filesVersion = await mlIDbStorage.getIndexVersion("files"); + if (filesVersion <= (await mlIDbStorage.getIndexVersion("people"))) { + return; + } + + // TODO: have faces addresable through fileId + faceId + // to avoid index based addressing, which is prone to wrong results + // one way could be to match nearest face within threshold in the file + const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext); + const allFaces = getAllFacesFromMap(allFacesMap); + + await FaceService.runFaceClustering(syncContext, allFaces); + await syncPeopleFromClusters(syncContext, allFacesMap, allFaces); + + await mlIDbStorage.setIndexVersion("people", filesVersion); +}; + +const syncPeopleFromClusters = async ( + syncContext: MLSyncContext, + allFacesMap: Map>, + allFaces: Array, +) => { + const clusters = syncContext.mlLibraryData.faceClusteringResults?.clusters; + if (!clusters || clusters.length < 1) { + return; + } + + for (const face of allFaces) { + face.personId = undefined; + } + await mlIDbStorage.clearAllPeople(); + for (const [index, cluster] of clusters.entries()) { + const faces = cluster.map((f) => allFaces[f]).filter((f) => f); + + // TODO: take default display face from last leaves of hdbscan clusters + const personFace = faces.reduce((best, face) => + face.detection.probability > best.detection.probability + ? face + : best, + ); + + if (personFace && !personFace.crop?.cacheKey) { + const file = await getLocalFile(personFace.fileId); + const imageBitmap = await fetchImageBitmap(file); + await FaceService.saveFaceCrop( + imageBitmap, + personFace, + syncContext, + ); + } + + const person: Person = { + id: index, + files: faces.map((f) => f.fileId), + displayFaceId: personFace?.id, + faceCropCacheKey: personFace?.crop?.cacheKey, + }; + + await mlIDbStorage.putPerson(person); + + faces.forEach((face) => { + face.personId = person.id; + }); + // log.info("Creating person: ", person, faces); + } + + await mlIDbStorage.updateFaces(allFacesMap); +}; + + +function getAllFacesFromMap(allFacesMap: Map>) { + const allFaces = [...allFacesMap.values()].flat(); + + return allFaces; +} From 8a071fd45b11528c88464b8d04260d797668ad87 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:24:38 +0530 Subject: [PATCH 21/47] Inline --- web/apps/photos/src/services/face/people.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index 8a0f0e969a..e6cea90078 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -13,7 +13,7 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => { // to avoid index based addressing, which is prone to wrong results // one way could be to match nearest face within threshold in the file const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext); - const allFaces = getAllFacesFromMap(allFacesMap); + const allFaces = [...allFacesMap.values()].flat(); await FaceService.runFaceClustering(syncContext, allFaces); await syncPeopleFromClusters(syncContext, allFacesMap, allFaces); @@ -72,10 +72,3 @@ const syncPeopleFromClusters = async ( await mlIDbStorage.updateFaces(allFacesMap); }; - - -function getAllFacesFromMap(allFacesMap: Map>) { - const allFaces = [...allFacesMap.values()].flat(); - - return allFaces; -} From ad684c46c347b99054d5d72d2f81376bcb367f34 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:43:58 +0530 Subject: [PATCH 22/47] Remove old file handling --- .../photos/src/services/face/face.worker.ts | 2 +- web/apps/photos/src/services/face/types.ts | 3 +- .../services/machineLearning/faceService.ts | 126 +----------------- .../machineLearning/machineLearningService.ts | 42 ++---- .../services/machineLearning/mlWorkManager.ts | 10 +- 5 files changed, 22 insertions(+), 161 deletions(-) diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index 8c9d094987..cc4d7017c9 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -19,7 +19,7 @@ export class DedicatedMLWorker implements MachineLearningWorker { enteFile: EnteFile, localFile: globalThis.File, ) { - return mlService.syncLocalFile(token, userID, enteFile, localFile); + mlService.syncLocalFile(token, userID, enteFile, localFile); } public async sync(token: string, userID: number) { diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index f7a88d9c63..9d2c4536f9 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -197,7 +197,6 @@ export interface MLSearchConfig { export interface MLSyncContext { token: string; userID: number; - shouldUpdateMLVersion: boolean; faceDetectionService: FaceDetectionService; faceCropService: FaceCropService; @@ -281,7 +280,7 @@ export interface MachineLearningWorker { userID: number, enteFile: EnteFile, localFile: globalThis.File, - ): Promise; + ); sync(token: string, userID: number): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 8b521add99..f5e83b8f5b 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -8,7 +8,6 @@ import { MLSyncContext, MLSyncFileContext, type FaceAlignment, - type Versioned, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { clusterFaces } from "../face/cluster"; @@ -24,36 +23,12 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !isDifferentOrOld( - oldMlFile?.faceDetectionMethod, - syncContext.faceDetectionService.method, - ) && - oldMlFile?.imageSource === "Original" - ) { - newMlFile.faces = oldMlFile?.faces?.map((existingFace) => ({ - id: existingFace.id, - fileId: existingFace.fileId, - detection: existingFace.detection, - })); - - newMlFile.imageSource = oldMlFile.imageSource; - newMlFile.imageDimensions = oldMlFile.imageDimensions; - newMlFile.faceDetectionMethod = oldMlFile.faceDetectionMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method; fileContext.newDetection = true; const imageBitmap = await fetchImageBitmapForContext(fileContext); - const timerId = `faceDetection-${fileContext.enteFile.id}`; - console.time(timerId); const faceDetections = await syncContext.faceDetectionService.detectFaces(imageBitmap); - console.timeEnd(timerId); - console.log("faceDetections: ", faceDetections?.length); - // TODO: reenable faces filtering based on width const detectedFaces = faceDetections?.map((detection) => { return { @@ -75,23 +50,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - // !syncContext.config.faceCrop.enabled || - !fileContext.newDetection && - !isDifferentOrOld( - oldMlFile?.faceCropMethod, - syncContext.faceCropService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.crop = oldMlFile.faces[index].crop; - } - newMlFile.faceCropMethod = oldMlFile.faceCropMethod; - return; - } - + const { newMlFile } = fileContext; const imageBitmap = await fetchImageBitmapForContext(fileContext); newMlFile.faceCropMethod = syncContext.faceCropService.method; @@ -104,24 +63,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ): Promise { - const { oldMlFile, newMlFile } = fileContext; - // TODO-ML(MR): - const method = { - value: "ArcFace", - version: 1, - }; - if ( - !fileContext.newDetection && - !isDifferentOrOld(oldMlFile?.faceAlignmentMethod, method) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.alignment = oldMlFile.faces[index].alignment; - } - newMlFile.faceAlignmentMethod = oldMlFile.faceAlignmentMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceAlignmentMethod = { value: "ArcFace", version: 1, @@ -159,22 +101,7 @@ class FaceService { fileContext: MLSyncFileContext, alignedFacesInput: Float32Array, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !fileContext.newAlignment && - !isDifferentOrOld( - oldMlFile?.faceEmbeddingMethod, - syncContext.faceEmbeddingService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - for (const [index, face] of newMlFile.faces.entries()) { - face.embedding = oldMlFile.faces[index].embedding; - } - newMlFile.faceEmbeddingMethod = oldMlFile.faceEmbeddingMethod; - return; - } - + const { newMlFile } = fileContext; newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method; // TODO: when not storing face crops, image will be needed to extract faces // fileContext.imageBitmap || @@ -193,17 +120,7 @@ class FaceService { syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) { - const { oldMlFile, newMlFile } = fileContext; - if ( - !fileContext.newAlignment && - !isDifferentOrOld( - oldMlFile?.faceEmbeddingMethod, - syncContext.faceEmbeddingService.method, - ) && - areFaceIdsSame(newMlFile.faces, oldMlFile?.faces) - ) { - return; - } + const { newMlFile } = fileContext; for (let i = 0; i < newMlFile.faces.length; i++) { const face = newMlFile.faces[i]; if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative @@ -298,39 +215,6 @@ class FaceService { export default new FaceService(); -export function areFaceIdsSame(ofFaces: Array, toFaces: Array) { - if ( - (ofFaces === null || ofFaces === undefined) && - (toFaces === null || toFaces === undefined) - ) { - return true; - } - return primitiveArrayEquals( - ofFaces?.map((f) => f.id), - toFaces?.map((f) => f.id), - ); -} - -function primitiveArrayEquals(a, b) { - return ( - Array.isArray(a) && - Array.isArray(b) && - a.length === b.length && - a.every((val, index) => val === b[index]) - ); -} - -export function isDifferentOrOld( - method: Versioned, - thanMethod: Versioned, -) { - return ( - !method || - method.value !== thanMethod.value || - method.version < thanMethod.version - ); -} - async function extractFaceImagesToFloat32( faceAlignments: Array, faceSize: number, diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 26d9e69dd9..058f2f001d 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -157,7 +157,6 @@ export class MLFactory { export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; - public shouldUpdateMLVersion: boolean; public faceDetectionService: FaceDetectionService; public faceCropService: FaceCropService; @@ -184,15 +183,9 @@ export class LocalMLSyncContext implements MLSyncContext { >; private enteWorkers: Array; - constructor( - token: string, - userID: number, - shouldUpdateMLVersion: boolean = true, - concurrency?: number, - ) { + constructor(token: string, userID: number, concurrency?: number) { this.token = token; this.userID = userID; - this.shouldUpdateMLVersion = shouldUpdateMLVersion; this.faceDetectionService = MLFactory.getFaceDetectionService("YoloFace"); @@ -424,7 +417,7 @@ class MachineLearningService { // TODO-ML(MR): Keep as promise for now. this.syncContext = new Promise((resolve) => { - resolve(new LocalMLSyncContext(token, userID, true)); + resolve(new LocalMLSyncContext(token, userID)); }); } else { log.info("reusing existing syncContext"); @@ -433,11 +426,12 @@ class MachineLearningService { } private async getLocalSyncContext(token: string, userID: number) { + // TODO-ML(MR): This is updating the file ML version. verify. if (!this.localSyncContext) { log.info("Creating localSyncContext"); // TODO-ML(MR): this.localSyncContext = new Promise((resolve) => { - resolve(new LocalMLSyncContext(token, userID, false)); + resolve(new LocalMLSyncContext(token, userID)); }); } else { log.info("reusing existing localSyncContext"); @@ -459,11 +453,11 @@ class MachineLearningService { userID: number, enteFile: EnteFile, localFile?: globalThis.File, - ): Promise { + ) { const syncContext = await this.getLocalSyncContext(token, userID); try { - const mlFileData = await this.syncFileWithErrorHandler( + await this.syncFileWithErrorHandler( syncContext, enteFile, localFile, @@ -473,10 +467,8 @@ class MachineLearningService { await this.closeLocalSyncContext(); } // await syncContext.dispose(); - return mlFileData; } catch (e) { console.error("Error while syncing local file: ", enteFile.id, e); - return e; } } @@ -484,7 +476,7 @@ class MachineLearningService { syncContext: MLSyncContext, enteFile: EnteFile, localFile?: globalThis.File, - ): Promise { + ) { try { console.log( `Indexing ${enteFile.title ?? ""} ${enteFile.id}`, @@ -533,22 +525,13 @@ class MachineLearningService { ) { console.log("Syncing for file" + enteFile.title); const fileContext: MLSyncFileContext = { enteFile, localFile }; - const oldMlFile = - (fileContext.oldMlFile = await this.getMLFileData(enteFile.id)) ?? - this.newMlData(enteFile.id); - if ( - fileContext.oldMlFile?.mlVersion === defaultMLVersion - // TODO: reset mlversion of all files when user changes image source - ) { - return fileContext.oldMlFile; + const oldMlFile = await this.getMLFileData(enteFile.id); + if (oldMlFile) { + return oldMlFile; } - const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); - if (syncContext.shouldUpdateMLVersion) { - newMlFile.mlVersion = defaultMLVersion; - } else if (fileContext.oldMlFile?.mlVersion) { - newMlFile.mlVersion = fileContext.oldMlFile.mlVersion; - } + const newMlFile = (fileContext.newMlFile = this.newMlData(enteFile.id)); + newMlFile.mlVersion = defaultMLVersion; try { await fetchImageBitmapForContext(fileContext); @@ -628,6 +611,7 @@ class MachineLearningService { public async syncIndex(syncContext: MLSyncContext) { await this.getMLLibraryData(syncContext); + // TODO-ML(MR): Ensure this doesn't run until fixed. await syncPeopleIndex(syncContext); await this.persistMLLibraryData(syncContext); diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 52fb43f9cf..8f58965f14 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -5,8 +5,8 @@ import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; -import mlIDbStorage from "services/face/db"; import { createFaceComlinkWorker } from "services/face"; +import mlIDbStorage from "services/face/db"; import type { DedicatedMLWorker } from "services/face/face.worker"; import { MLSyncResult } from "services/face/types"; import { EnteFile } from "types/file"; @@ -232,19 +232,13 @@ class MLWorkManager { } public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) { - const result = await this.liveSyncQueue.add(async () => { + await this.liveSyncQueue.add(async () => { this.stopSyncJob(); const token = getToken(); const userID = getUserID(); const mlWorker = await this.getLiveSyncWorker(); return mlWorker.syncLocalFile(token, userID, enteFile, localFile); }); - - if (result instanceof Error) { - // TODO: redirect/refresh to gallery in case of session_expired - // may not be required as uploader should anyways take care of this - console.error("Error while syncing local file: ", result); - } } // Sync Job From adda781dcc1ecc43a7b7bd0f82e1be3239932913 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:54:15 +0530 Subject: [PATCH 23/47] Fix initial run --- .../src/services/machineLearning/machineLearningService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 058f2f001d..bf663ee3e2 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -523,10 +523,10 @@ class MachineLearningService { enteFile: EnteFile, localFile?: globalThis.File, ) { - console.log("Syncing for file" + enteFile.title); + log.debug(() => ({ a: "Syncing file", enteFile })); const fileContext: MLSyncFileContext = { enteFile, localFile }; const oldMlFile = await this.getMLFileData(enteFile.id); - if (oldMlFile) { + if (oldMlFile && oldMlFile.mlVersion) { return oldMlFile; } From fca2d460f98c5b2298124e6ed29dc20aafa00300 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 12:55:05 +0530 Subject: [PATCH 24/47] Disable clustering --- .../src/services/machineLearning/machineLearningService.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index bf663ee3e2..302a80b53a 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -258,11 +258,10 @@ class MachineLearningService { await this.syncFiles(syncContext); } - // TODO: running index before all files are on latest ml version - // may be need to just take synced files on latest ml version for indexing if ( syncContext.outOfSyncFiles.length <= 0 || - (syncContext.nSyncedFiles === batchSize && Math.random() < 0.2) + // TODO-ML(MR): Forced disable. + (syncContext.nSyncedFiles === batchSize && Math.random() < 0) ) { await this.syncIndex(syncContext); } From db05afb9ffbbd76baa813baf41b23e4da76fc6e0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:00:32 +0530 Subject: [PATCH 25/47] Inline and move --- web/apps/photos/src/services/face/crop.ts | 32 ++++++++++ web/apps/photos/src/services/face/people.ts | 6 +- .../machineLearning/arcfaceCropService.ts | 60 ------------------- .../services/machineLearning/faceService.ts | 21 ++----- .../machineLearning/machineLearningService.ts | 14 +---- 5 files changed, 40 insertions(+), 93 deletions(-) create mode 100644 web/apps/photos/src/services/face/crop.ts delete mode 100644 web/apps/photos/src/services/machineLearning/arcfaceCropService.ts diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts new file mode 100644 index 0000000000..acd49228eb --- /dev/null +++ b/web/apps/photos/src/services/face/crop.ts @@ -0,0 +1,32 @@ +import { Box, enlargeBox } from "services/face/geom"; +import { FaceCrop, FaceDetection } from "services/face/types"; +import { cropWithRotation } from "utils/image"; +import { faceAlignment } from "./align"; + +export const getFaceCrop = ( + imageBitmap: ImageBitmap, + faceDetection: FaceDetection, +): FaceCrop => { + const alignment = faceAlignment(faceDetection); + + const padding = 0.25; + const maxSize = 256; + + const alignmentBox = new Box({ + x: alignment.center.x - alignment.size / 2, + y: alignment.center.y - alignment.size / 2, + width: alignment.size, + height: alignment.size, + }).round(); + const scaleForPadding = 1 + padding * 2; + const paddedBox = enlargeBox(alignmentBox, scaleForPadding).round(); + const faceImageBitmap = cropWithRotation(imageBitmap, paddedBox, 0, { + width: maxSize, + height: maxSize, + }); + + return { + image: faceImageBitmap, + imageBox: paddedBox, + }; +}; diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index e6cea90078..081962935a 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -48,11 +48,7 @@ const syncPeopleFromClusters = async ( if (personFace && !personFace.crop?.cacheKey) { const file = await getLocalFile(personFace.fileId); const imageBitmap = await fetchImageBitmap(file); - await FaceService.saveFaceCrop( - imageBitmap, - personFace, - syncContext, - ); + await FaceService.saveFaceCrop(imageBitmap, personFace); } const person: Person = { diff --git a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts b/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts deleted file mode 100644 index 81f2d4de5e..0000000000 --- a/web/apps/photos/src/services/machineLearning/arcfaceCropService.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Box, enlargeBox } from "services/face/geom"; -import { - FaceAlignment, - FaceCrop, - FaceCropMethod, - FaceCropService, - FaceDetection, - Versioned, -} from "services/face/types"; -import { cropWithRotation } from "utils/image"; -import { faceAlignment } from "../face/align"; - -class ArcFaceCropService implements FaceCropService { - public method: Versioned; - - constructor() { - this.method = { - value: "ArcFace", - version: 1, - }; - } - - public async getFaceCrop( - imageBitmap: ImageBitmap, - faceDetection: FaceDetection, - ): Promise { - const alignment = faceAlignment(faceDetection); - const faceCrop = getFaceCrop(imageBitmap, alignment); - - return faceCrop; - } -} - -export default new ArcFaceCropService(); - -export function getFaceCrop( - imageBitmap: ImageBitmap, - alignment: FaceAlignment, -): FaceCrop { - const padding = 0.25; - const maxSize = 256; - - const alignmentBox = new Box({ - x: alignment.center.x - alignment.size / 2, - y: alignment.center.y - alignment.size / 2, - width: alignment.size, - height: alignment.size, - }).round(); - const scaleForPadding = 1 + padding * 2; - const paddedBox = enlargeBox(alignmentBox, scaleForPadding).round(); - const faceImageBitmap = cropWithRotation(imageBitmap, paddedBox, 0, { - width: maxSize, - height: maxSize, - }); - - return { - image: faceImageBitmap, - imageBox: paddedBox, - }; -} diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index f5e83b8f5b..c6c061af14 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -11,6 +11,7 @@ import { } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { clusterFaces } from "../face/cluster"; +import { getFaceCrop } from "../face/crop"; import { fetchImageBitmap, fetchImageBitmapForContext, @@ -55,7 +56,7 @@ class FaceService { newMlFile.faceCropMethod = syncContext.faceCropService.method; for (const face of newMlFile.faces) { - await this.saveFaceCrop(imageBitmap, face, syncContext); + await this.saveFaceCrop(imageBitmap, face); } } @@ -132,15 +133,8 @@ class FaceService { } } - async saveFaceCrop( - imageBitmap: ImageBitmap, - face: Face, - syncContext: MLSyncContext, - ) { - const faceCrop = await syncContext.faceCropService.getFaceCrop( - imageBitmap, - face.detection, - ); + async saveFaceCrop(imageBitmap: ImageBitmap, face: Face) { + const faceCrop = getFaceCrop(imageBitmap, face.detection); const blob = await imageBitmapToBlob(faceCrop.image); @@ -197,10 +191,7 @@ class FaceService { // }; } - public async regenerateFaceCrop( - syncContext: MLSyncContext, - faceID: string, - ) { + public async regenerateFaceCrop(faceID: string) { const fileID = Number(faceID.split("-")[0]); const personFace = await mlIDbStorage.getFace(fileID, faceID); if (!personFace) { @@ -209,7 +200,7 @@ class FaceService { const file = await getLocalFile(personFace.fileId); const imageBitmap = await fetchImageBitmap(file); - return await this.saveFaceCrop(imageBitmap, personFace, syncContext); + return await this.saveFaceCrop(imageBitmap, personFace); } } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 302a80b53a..ec7f97807a 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -15,7 +15,6 @@ import { BlurDetectionMethod, BlurDetectionService, Face, - FaceCropMethod, FaceCropService, FaceDetection, FaceDetectionMethod, @@ -34,7 +33,6 @@ import { import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import arcfaceCropService from "./arcfaceCropService"; import FaceService from "./faceService"; import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; @@ -125,14 +123,6 @@ export class MLFactory { throw Error("Unknon face detection method: " + method); } - public static getFaceCropService(method: FaceCropMethod) { - if (method === "ArcFace") { - return arcfaceCropService; - } - - throw Error("Unknon face crop method: " + method); - } - public static getBlurDetectionService( method: BlurDetectionMethod, ): BlurDetectionService { @@ -189,7 +179,6 @@ export class LocalMLSyncContext implements MLSyncContext { this.faceDetectionService = MLFactory.getFaceDetectionService("YoloFace"); - this.faceCropService = MLFactory.getFaceCropService("ArcFace"); this.blurDetectionService = MLFactory.getBlurDetectionService("Laplacian"); this.faceEmbeddingService = @@ -288,8 +277,7 @@ class MachineLearningService { faceID: string, ) { await downloadManager.init(APPS.PHOTOS, { token }); - const syncContext = await this.getSyncContext(token, userID); - return FaceService.regenerateFaceCrop(syncContext, faceID); + return FaceService.regenerateFaceCrop(faceID); } private newMlData(fileId: number) { From 7160ae700f7786d953659dda9f53d2acf1fd02cc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:07:55 +0530 Subject: [PATCH 26/47] Inline --- web/apps/photos/src/services/face/types.ts | 6 - .../services/machineLearning/faceService.ts | 6 +- .../laplacianBlurDetectionService.ts | 284 ++++++++---------- .../machineLearning/machineLearningService.ts | 16 - 4 files changed, 135 insertions(+), 177 deletions(-) diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index 9d2c4536f9..e6321cb76a 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -201,7 +201,6 @@ export interface MLSyncContext { faceDetectionService: FaceDetectionService; faceCropService: FaceCropService; faceEmbeddingService: FaceEmbeddingService; - blurDetectionService: BlurDetectionService; localFilesMap: Map; outOfSyncFiles: EnteFile[]; @@ -267,11 +266,6 @@ export interface FaceEmbeddingService { getFaceEmbeddings(faceImages: Float32Array): Promise>; } -export interface BlurDetectionService { - method: Versioned; - detectBlur(alignedFaces: Float32Array, faces: Face[]): number[]; -} - export interface MachineLearningWorker { closeLocalSyncContext(): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index c6c061af14..73e3f4c6e4 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -18,6 +18,7 @@ import { getFaceId, getLocalFile, } from "../face/image"; +import { detectBlur } from "./laplacianBlurDetectionService"; class FaceService { async syncFileFaceDetections( @@ -85,10 +86,7 @@ class FaceService { syncContext.faceEmbeddingService.faceSize, imageBitmap, ); - const blurValues = syncContext.blurDetectionService.detectBlur( - faceImages, - newMlFile.faces, - ); + const blurValues = detectBlur(faceImages, newMlFile.faces); newMlFile.faces.forEach((f, i) => (f.blurValue = blurValues[i])); imageBitmap.close(); diff --git a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts index a4d8000cf0..71641cdb6c 100644 --- a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts @@ -1,176 +1,158 @@ -import { - BlurDetectionMethod, - BlurDetectionService, - Face, - Versioned, -} from "services/face/types"; +import { Face } from "services/face/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; import { mobileFaceNetFaceSize } from "./mobileFaceNetEmbeddingService"; -class LaplacianBlurDetectionService implements BlurDetectionService { - public method: Versioned; - - public constructor() { - this.method = { - value: "Laplacian", - version: 1, - }; - } - - public detectBlur(alignedFaces: Float32Array, faces: Face[]): number[] { - const numFaces = Math.round( - alignedFaces.length / - (mobileFaceNetFaceSize * mobileFaceNetFaceSize * 3), +/** + * Laplacian blur detection. + */ +export const detectBlur = ( + alignedFaces: Float32Array, + faces: Face[], +): number[] => { + const numFaces = Math.round( + alignedFaces.length / + (mobileFaceNetFaceSize * mobileFaceNetFaceSize * 3), + ); + const blurValues: number[] = []; + for (let i = 0; i < numFaces; i++) { + const face = faces[i]; + const direction = faceDirection(face); + const faceImage = createGrayscaleIntMatrixFromNormalized2List( + alignedFaces, + i, ); - const blurValues: number[] = []; - for (let i = 0; i < numFaces; i++) { - const face = faces[i]; - const direction = getFaceDirection(face); - const faceImage = createGrayscaleIntMatrixFromNormalized2List( - alignedFaces, - i, - ); - const laplacian = this.applyLaplacian(faceImage, direction); - const variance = this.calculateVariance(laplacian); - blurValues.push(variance); - } - return blurValues; + const laplacian = applyLaplacian(faceImage, direction); + const variance = calculateVariance(laplacian); + blurValues.push(variance); } + return blurValues; +}; - private calculateVariance(matrix: number[][]): number { - const numRows = matrix.length; - const numCols = matrix[0].length; - const totalElements = numRows * numCols; +const calculateVariance = (matrix: number[][]): number => { + const numRows = matrix.length; + const numCols = matrix[0].length; + const totalElements = numRows * numCols; - // Calculate the mean - let mean: number = 0; - matrix.forEach((row) => { - row.forEach((value) => { - mean += value; - }); + // Calculate the mean + let mean: number = 0; + matrix.forEach((row) => { + row.forEach((value) => { + mean += value; }); - mean /= totalElements; + }); + mean /= totalElements; - // Calculate the variance - let variance: number = 0; - matrix.forEach((row) => { - row.forEach((value) => { - const diff: number = value - mean; - variance += diff * diff; - }); + // Calculate the variance + let variance: number = 0; + matrix.forEach((row) => { + row.forEach((value) => { + const diff: number = value - mean; + variance += diff * diff; }); - variance /= totalElements; + }); + variance /= totalElements; - return variance; + return variance; +}; + +const padImage = ( + image: number[][], + removeSideColumns: number = 56, + direction: FaceDirection = "straight", +): number[][] => { + // Exception is removeSideColumns is not even + if (removeSideColumns % 2 != 0) { + throw new Error("removeSideColumns must be even"); } + const numRows = image.length; + const numCols = image[0].length; + const paddedNumCols = numCols + 2 - removeSideColumns; + const paddedNumRows = numRows + 2; - private padImage( - image: number[][], - removeSideColumns: number = 56, - direction: FaceDirection = "straight", - ): number[][] { - // Exception is removeSideColumns is not even - if (removeSideColumns % 2 != 0) { - throw new Error("removeSideColumns must be even"); - } - const numRows = image.length; - const numCols = image[0].length; - const paddedNumCols = numCols + 2 - removeSideColumns; - const paddedNumRows = numRows + 2; + // Create a new matrix with extra padding + const paddedImage: number[][] = Array.from({ length: paddedNumRows }, () => + new Array(paddedNumCols).fill(0), + ); - // Create a new matrix with extra padding - const paddedImage: number[][] = Array.from( - { length: paddedNumRows }, - () => new Array(paddedNumCols).fill(0), - ); - - // Copy original image into the center of the padded image - if (direction === "straight") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = - image[i][j + Math.round(removeSideColumns / 2)]; - } - } - } // If the face is facing left, we only take the right side of the face image - else if (direction === "left") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns]; - } - } - } // If the face is facing right, we only take the left side of the face image - else if (direction === "right") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = image[i][j]; - } - } - } - - // Reflect padding - // Top and bottom rows - for (let j = 1; j <= paddedNumCols - 2; j++) { - paddedImage[0][j] = paddedImage[2][j]; // Top row - paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row - } - // Left and right columns - for (let i = 0; i < numRows + 2; i++) { - paddedImage[i][0] = paddedImage[i][2]; // Left column - paddedImage[i][paddedNumCols - 1] = - paddedImage[i][paddedNumCols - 3]; // Right column - } - - return paddedImage; - } - - private applyLaplacian( - image: number[][], - direction: FaceDirection = "straight", - ): number[][] { - const paddedImage: number[][] = this.padImage( - image, - undefined, - direction, - ); - const numRows = paddedImage.length - 2; - const numCols = paddedImage[0].length - 2; - - // Create an output image initialized to 0 - const outputImage: number[][] = Array.from({ length: numRows }, () => - new Array(numCols).fill(0), - ); - - // Define the Laplacian kernel - const kernel: number[][] = [ - [0, 1, 0], - [1, -4, 1], - [0, 1, 0], - ]; - - // Apply the kernel to each pixel + // Copy original image into the center of the padded image + if (direction === "straight") { for (let i = 0; i < numRows; i++) { - for (let j = 0; j < numCols; j++) { - let sum = 0; - for (let ki = 0; ki < 3; ki++) { - for (let kj = 0; kj < 3; kj++) { - sum += paddedImage[i + ki][j + kj] * kernel[ki][kj]; - } - } - // Adjust the output value if necessary (e.g., clipping) - outputImage[i][j] = sum; + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = + image[i][j + Math.round(removeSideColumns / 2)]; + } + } + } // If the face is facing left, we only take the right side of the face image + else if (direction === "left") { + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns]; + } + } + } // If the face is facing right, we only take the left side of the face image + else if (direction === "right") { + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = image[i][j]; } } - - return outputImage; } -} -export default new LaplacianBlurDetectionService(); + // Reflect padding + // Top and bottom rows + for (let j = 1; j <= paddedNumCols - 2; j++) { + paddedImage[0][j] = paddedImage[2][j]; // Top row + paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row + } + // Left and right columns + for (let i = 0; i < numRows + 2; i++) { + paddedImage[i][0] = paddedImage[i][2]; // Left column + paddedImage[i][paddedNumCols - 1] = paddedImage[i][paddedNumCols - 3]; // Right column + } + + return paddedImage; +}; + +const applyLaplacian = ( + image: number[][], + direction: FaceDirection = "straight", +): number[][] => { + const paddedImage: number[][] = padImage(image, undefined, direction); + const numRows = paddedImage.length - 2; + const numCols = paddedImage[0].length - 2; + + // Create an output image initialized to 0 + const outputImage: number[][] = Array.from({ length: numRows }, () => + new Array(numCols).fill(0), + ); + + // Define the Laplacian kernel + const kernel: number[][] = [ + [0, 1, 0], + [1, -4, 1], + [0, 1, 0], + ]; + + // Apply the kernel to each pixel + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < numCols; j++) { + let sum = 0; + for (let ki = 0; ki < 3; ki++) { + for (let kj = 0; kj < 3; kj++) { + sum += paddedImage[i + ki][j + kj] * kernel[ki][kj]; + } + } + // Adjust the output value if necessary (e.g., clipping) + outputImage[i][j] = sum; + } + } + + return outputImage; +}; type FaceDirection = "left" | "right" | "straight"; -const getFaceDirection = (face: Face): FaceDirection => { +const faceDirection = (face: Face): FaceDirection => { const landmarks = face.detection.landmarks; const leftEye = landmarks[0]; const rightEye = landmarks[1]; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index ec7f97807a..30866ed3ce 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -12,8 +12,6 @@ import downloadManager from "services/download"; import { putEmbedding } from "services/embeddingService"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { - BlurDetectionMethod, - BlurDetectionService, Face, FaceCropService, FaceDetection, @@ -34,7 +32,6 @@ import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import FaceService from "./faceService"; -import laplacianBlurDetectionService from "./laplacianBlurDetectionService"; import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import { fetchImageBitmapForContext } from "../face/image"; @@ -123,16 +120,6 @@ export class MLFactory { throw Error("Unknon face detection method: " + method); } - public static getBlurDetectionService( - method: BlurDetectionMethod, - ): BlurDetectionService { - if (method === "Laplacian") { - return laplacianBlurDetectionService; - } - - throw Error("Unknon blur detection method: " + method); - } - public static getFaceEmbeddingService( method: FaceEmbeddingMethod, ): FaceEmbeddingService { @@ -150,7 +137,6 @@ export class LocalMLSyncContext implements MLSyncContext { public faceDetectionService: FaceDetectionService; public faceCropService: FaceCropService; - public blurDetectionService: BlurDetectionService; public faceEmbeddingService: FaceEmbeddingService; public localFilesMap: Map; @@ -179,8 +165,6 @@ export class LocalMLSyncContext implements MLSyncContext { this.faceDetectionService = MLFactory.getFaceDetectionService("YoloFace"); - this.blurDetectionService = - MLFactory.getBlurDetectionService("Laplacian"); this.faceEmbeddingService = MLFactory.getFaceEmbeddingService("MobileFaceNet"); From 569808c2916fc3986add2329c41d6c6f3e669d6a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:16:20 +0530 Subject: [PATCH 27/47] Rearrange --- .../laplacianBlurDetectionService.ts | 246 +++++++++--------- 1 file changed, 120 insertions(+), 126 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts index 71641cdb6c..d4c35c97f1 100644 --- a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts +++ b/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts @@ -22,134 +22,11 @@ export const detectBlur = ( i, ); const laplacian = applyLaplacian(faceImage, direction); - const variance = calculateVariance(laplacian); - blurValues.push(variance); + blurValues.push(matrixVariance(laplacian)); } return blurValues; }; -const calculateVariance = (matrix: number[][]): number => { - const numRows = matrix.length; - const numCols = matrix[0].length; - const totalElements = numRows * numCols; - - // Calculate the mean - let mean: number = 0; - matrix.forEach((row) => { - row.forEach((value) => { - mean += value; - }); - }); - mean /= totalElements; - - // Calculate the variance - let variance: number = 0; - matrix.forEach((row) => { - row.forEach((value) => { - const diff: number = value - mean; - variance += diff * diff; - }); - }); - variance /= totalElements; - - return variance; -}; - -const padImage = ( - image: number[][], - removeSideColumns: number = 56, - direction: FaceDirection = "straight", -): number[][] => { - // Exception is removeSideColumns is not even - if (removeSideColumns % 2 != 0) { - throw new Error("removeSideColumns must be even"); - } - const numRows = image.length; - const numCols = image[0].length; - const paddedNumCols = numCols + 2 - removeSideColumns; - const paddedNumRows = numRows + 2; - - // Create a new matrix with extra padding - const paddedImage: number[][] = Array.from({ length: paddedNumRows }, () => - new Array(paddedNumCols).fill(0), - ); - - // Copy original image into the center of the padded image - if (direction === "straight") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = - image[i][j + Math.round(removeSideColumns / 2)]; - } - } - } // If the face is facing left, we only take the right side of the face image - else if (direction === "left") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns]; - } - } - } // If the face is facing right, we only take the left side of the face image - else if (direction === "right") { - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < paddedNumCols - 2; j++) { - paddedImage[i + 1][j + 1] = image[i][j]; - } - } - } - - // Reflect padding - // Top and bottom rows - for (let j = 1; j <= paddedNumCols - 2; j++) { - paddedImage[0][j] = paddedImage[2][j]; // Top row - paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row - } - // Left and right columns - for (let i = 0; i < numRows + 2; i++) { - paddedImage[i][0] = paddedImage[i][2]; // Left column - paddedImage[i][paddedNumCols - 1] = paddedImage[i][paddedNumCols - 3]; // Right column - } - - return paddedImage; -}; - -const applyLaplacian = ( - image: number[][], - direction: FaceDirection = "straight", -): number[][] => { - const paddedImage: number[][] = padImage(image, undefined, direction); - const numRows = paddedImage.length - 2; - const numCols = paddedImage[0].length - 2; - - // Create an output image initialized to 0 - const outputImage: number[][] = Array.from({ length: numRows }, () => - new Array(numCols).fill(0), - ); - - // Define the Laplacian kernel - const kernel: number[][] = [ - [0, 1, 0], - [1, -4, 1], - [0, 1, 0], - ]; - - // Apply the kernel to each pixel - for (let i = 0; i < numRows; i++) { - for (let j = 0; j < numCols; j++) { - let sum = 0; - for (let ki = 0; ki < 3; ki++) { - for (let kj = 0; kj < 3; kj++) { - sum += paddedImage[i + ki][j + kj] * kernel[ki][kj]; - } - } - // Adjust the output value if necessary (e.g., clipping) - outputImage[i][j] = sum; - } - } - - return outputImage; -}; - type FaceDirection = "left" | "right" | "straight"; const faceDirection = (face: Face): FaceDirection => { @@ -181,13 +58,130 @@ const faceDirection = (face: Face): FaceDirection => { const noseCloseToRightEye = Math.abs(nose.x - rightEye.x) < 0.2 * eyeDistanceX; - // if (faceIsUpright && (noseStickingOutLeft || noseCloseToLeftEye)) { if (noseStickingOutLeft || (faceIsUpright && noseCloseToLeftEye)) { return "left"; - // } else if (faceIsUpright && (noseStickingOutRight || noseCloseToRightEye)) { } else if (noseStickingOutRight || (faceIsUpright && noseCloseToRightEye)) { return "right"; } return "straight"; }; + +/** + * Return a new image by applying a Laplacian blur kernel to each pixel. + */ +const applyLaplacian = ( + image: number[][], + direction: FaceDirection, +): number[][] => { + const paddedImage: number[][] = padImage(image, direction); + const numRows = paddedImage.length - 2; + const numCols = paddedImage[0].length - 2; + + // Create an output image initialized to 0. + const outputImage: number[][] = Array.from({ length: numRows }, () => + new Array(numCols).fill(0), + ); + + // Define the Laplacian kernel. + const kernel: number[][] = [ + [0, 1, 0], + [1, -4, 1], + [0, 1, 0], + ]; + + // Apply the kernel to each pixel + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < numCols; j++) { + let sum = 0; + for (let ki = 0; ki < 3; ki++) { + for (let kj = 0; kj < 3; kj++) { + sum += paddedImage[i + ki][j + kj] * kernel[ki][kj]; + } + } + // Adjust the output value if necessary (e.g., clipping). + outputImage[i][j] = sum; + } + } + + return outputImage; +}; + +const padImage = (image: number[][], direction: FaceDirection): number[][] => { + const removeSideColumns = 56; /* must be even */ + + const numRows = image.length; + const numCols = image[0].length; + const paddedNumCols = numCols + 2 - removeSideColumns; + const paddedNumRows = numRows + 2; + + // Create a new matrix with extra padding. + const paddedImage: number[][] = Array.from({ length: paddedNumRows }, () => + new Array(paddedNumCols).fill(0), + ); + + if (direction === "straight") { + // Copy original image into the center of the padded image. + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = + image[i][j + Math.round(removeSideColumns / 2)]; + } + } + } else if (direction === "left") { + // If the face is facing left, we only take the right side of the face image. + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = image[i][j + removeSideColumns]; + } + } + } else if (direction === "right") { + // If the face is facing right, we only take the left side of the face image. + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < paddedNumCols - 2; j++) { + paddedImage[i + 1][j + 1] = image[i][j]; + } + } + } + + // Reflect padding + // Top and bottom rows + for (let j = 1; j <= paddedNumCols - 2; j++) { + paddedImage[0][j] = paddedImage[2][j]; // Top row + paddedImage[numRows + 1][j] = paddedImage[numRows - 1][j]; // Bottom row + } + // Left and right columns + for (let i = 0; i < numRows + 2; i++) { + paddedImage[i][0] = paddedImage[i][2]; // Left column + paddedImage[i][paddedNumCols - 1] = paddedImage[i][paddedNumCols - 3]; // Right column + } + + return paddedImage; +}; + +const matrixVariance = (matrix: number[][]): number => { + const numRows = matrix.length; + const numCols = matrix[0].length; + const totalElements = numRows * numCols; + + // Calculate the mean. + let mean: number = 0; + matrix.forEach((row) => { + row.forEach((value) => { + mean += value; + }); + }); + mean /= totalElements; + + // Calculate the variance. + let variance: number = 0; + matrix.forEach((row) => { + row.forEach((value) => { + const diff: number = value - mean; + variance += diff * diff; + }); + }); + variance /= totalElements; + + return variance; +}; From d8f707841842489b7deb6fe13a5569d93121ca26 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:16:53 +0530 Subject: [PATCH 28/47] Move --- .../laplacianBlurDetectionService.ts => face/detect-blur.ts} | 2 +- web/apps/photos/src/services/machineLearning/faceService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename web/apps/photos/src/services/{machineLearning/laplacianBlurDetectionService.ts => face/detect-blur.ts} (98%) diff --git a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts b/web/apps/photos/src/services/face/detect-blur.ts similarity index 98% rename from web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts rename to web/apps/photos/src/services/face/detect-blur.ts index d4c35c97f1..2c6083d6e0 100644 --- a/web/apps/photos/src/services/machineLearning/laplacianBlurDetectionService.ts +++ b/web/apps/photos/src/services/face/detect-blur.ts @@ -1,6 +1,6 @@ import { Face } from "services/face/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; -import { mobileFaceNetFaceSize } from "./mobileFaceNetEmbeddingService"; +import { mobileFaceNetFaceSize } from "../machineLearning/mobileFaceNetEmbeddingService"; /** * Laplacian blur detection. diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 73e3f4c6e4..843d2106ab 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -10,6 +10,7 @@ import { type FaceAlignment, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { detectBlur } from "../face/detect-blur"; import { clusterFaces } from "../face/cluster"; import { getFaceCrop } from "../face/crop"; import { @@ -18,7 +19,6 @@ import { getFaceId, getLocalFile, } from "../face/image"; -import { detectBlur } from "./laplacianBlurDetectionService"; class FaceService { async syncFileFaceDetections( From 36af1cfacda6840da702c5ebba6cdf796dfd47f7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:19:30 +0530 Subject: [PATCH 29/47] Move --- .../yoloFaceDetectionService.ts => face/detect-face.ts} | 0 .../src/services/machineLearning/machineLearningService.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename web/apps/photos/src/services/{machineLearning/yoloFaceDetectionService.ts => face/detect-face.ts} (100%) diff --git a/web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts b/web/apps/photos/src/services/face/detect-face.ts similarity index 100% rename from web/apps/photos/src/services/machineLearning/yoloFaceDetectionService.ts rename to web/apps/photos/src/services/face/detect-face.ts diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 30866ed3ce..72c34694ae 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -36,7 +36,7 @@ import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; import { fetchImageBitmapForContext } from "../face/image"; import { syncPeopleIndex } from "../face/people"; -import yoloFaceDetectionService from "./yoloFaceDetectionService"; +import yoloFaceDetectionService from "../face/detect-face"; /** * TODO-ML(MR): What and why. From 839b4c04a9fbec9ce4f082314776def4f85a5ff7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:25:47 +0530 Subject: [PATCH 30/47] Unclass --- .../photos/src/services/face/detect-face.ts | 417 +++++++++--------- web/apps/photos/src/services/face/types.ts | 11 - .../services/machineLearning/faceService.ts | 20 +- .../machineLearning/machineLearningService.ts | 21 +- 4 files changed, 212 insertions(+), 257 deletions(-) diff --git a/web/apps/photos/src/services/face/detect-face.ts b/web/apps/photos/src/services/face/detect-face.ts index e9e4fda622..5c6b633885 100644 --- a/web/apps/photos/src/services/face/detect-face.ts +++ b/web/apps/photos/src/services/face/detect-face.ts @@ -7,12 +7,7 @@ import { boxFromBoundingBox, newBox, } from "services/face/geom"; -import { - FaceDetection, - FaceDetectionMethod, - FaceDetectionService, - Versioned, -} from "services/face/types"; +import { FaceDetection } from "services/face/types"; import { Matrix, applyToPoint, @@ -26,222 +21,208 @@ import { normalizePixelBetween0And1, } from "utils/image"; -class YoloFaceDetectionService implements FaceDetectionService { - public method: Versioned; - - public constructor() { - this.method = { - value: "YoloFace", - version: 1, - }; - } - - public async detectFaces( - imageBitmap: ImageBitmap, - ): Promise> { - const maxFaceDistancePercent = Math.sqrt(2) / 100; - const maxFaceDistance = imageBitmap.width * maxFaceDistancePercent; - const preprocessResult = - this.preprocessImageBitmapToFloat32ChannelsFirst( - imageBitmap, - 640, - 640, - ); - const data = preprocessResult.data; - const resized = preprocessResult.newSize; - const outputData = await workerBridge.detectFaces(data); - const faces = this.getFacesFromYoloOutput( - outputData as Float32Array, - 0.7, - ); - const inBox = newBox(0, 0, resized.width, resized.height); - const toBox = newBox(0, 0, imageBitmap.width, imageBitmap.height); - const transform = computeTransformToBox(inBox, toBox); - const faceDetections: Array = faces?.map((f) => { - const box = transformBox(f.box, transform); - const normLandmarks = f.landmarks; - const landmarks = transformPoints(normLandmarks, transform); - return { - box, - landmarks, - probability: f.probability as number, - } as FaceDetection; - }); - return removeDuplicateDetections(faceDetections, maxFaceDistance); - } - - private preprocessImageBitmapToFloat32ChannelsFirst( - imageBitmap: ImageBitmap, - requiredWidth: number, - requiredHeight: number, - maintainAspectRatio: boolean = true, - normFunction: ( - pixelValue: number, - ) => number = normalizePixelBetween0And1, - ) { - // Create an OffscreenCanvas and set its size - const offscreenCanvas = new OffscreenCanvas( - imageBitmap.width, - imageBitmap.height, - ); - const ctx = offscreenCanvas.getContext("2d"); - ctx.drawImage(imageBitmap, 0, 0, imageBitmap.width, imageBitmap.height); - const imageData = ctx.getImageData( - 0, - 0, - imageBitmap.width, - imageBitmap.height, - ); - const pixelData = imageData.data; - - let scaleW = requiredWidth / imageBitmap.width; - let scaleH = requiredHeight / imageBitmap.height; - if (maintainAspectRatio) { - const scale = Math.min( - requiredWidth / imageBitmap.width, - requiredHeight / imageBitmap.height, - ); - scaleW = scale; - scaleH = scale; - } - const scaledWidth = clamp( - Math.round(imageBitmap.width * scaleW), - 0, - requiredWidth, - ); - const scaledHeight = clamp( - Math.round(imageBitmap.height * scaleH), - 0, - requiredHeight, - ); - - const processedImage = new Float32Array( - 1 * 3 * requiredWidth * requiredHeight, - ); - - // Populate the Float32Array with normalized pixel values - let pixelIndex = 0; - const channelOffsetGreen = requiredHeight * requiredWidth; - const channelOffsetBlue = 2 * requiredHeight * requiredWidth; - for (let h = 0; h < requiredHeight; h++) { - for (let w = 0; w < requiredWidth; w++) { - let pixel: { - r: number; - g: number; - b: number; - }; - if (w >= scaledWidth || h >= scaledHeight) { - pixel = { r: 114, g: 114, b: 114 }; - } else { - pixel = getPixelBilinear( - w / scaleW, - h / scaleH, - pixelData, - imageBitmap.width, - imageBitmap.height, - ); - } - processedImage[pixelIndex] = normFunction(pixel.r); - processedImage[pixelIndex + channelOffsetGreen] = normFunction( - pixel.g, - ); - processedImage[pixelIndex + channelOffsetBlue] = normFunction( - pixel.b, - ); - pixelIndex++; - } - } - - return { - data: processedImage, - originalSize: { - width: imageBitmap.width, - height: imageBitmap.height, - }, - newSize: { width: scaledWidth, height: scaledHeight }, - }; - } - - // The rowOutput is a Float32Array of shape [25200, 16], where each row represents a bounding box. - private getFacesFromYoloOutput( - rowOutput: Float32Array, - minScore: number, - ): Array { - const faces: Array = []; - // iterate over each row - for (let i = 0; i < rowOutput.length; i += 16) { - const score = rowOutput[i + 4]; - if (score < minScore) { - continue; - } - // The first 4 values represent the bounding box's coordinates (x1, y1, x2, y2) - const xCenter = rowOutput[i]; - const yCenter = rowOutput[i + 1]; - const width = rowOutput[i + 2]; - const height = rowOutput[i + 3]; - const xMin = xCenter - width / 2.0; // topLeft - const yMin = yCenter - height / 2.0; // topLeft - - const leftEyeX = rowOutput[i + 5]; - const leftEyeY = rowOutput[i + 6]; - const rightEyeX = rowOutput[i + 7]; - const rightEyeY = rowOutput[i + 8]; - const noseX = rowOutput[i + 9]; - const noseY = rowOutput[i + 10]; - const leftMouthX = rowOutput[i + 11]; - const leftMouthY = rowOutput[i + 12]; - const rightMouthX = rowOutput[i + 13]; - const rightMouthY = rowOutput[i + 14]; - - const box = new Box({ - x: xMin, - y: yMin, - width: width, - height: height, - }); - const probability = score as number; - const landmarks = [ - new Point(leftEyeX, leftEyeY), - new Point(rightEyeX, rightEyeY), - new Point(noseX, noseY), - new Point(leftMouthX, leftMouthY), - new Point(rightMouthX, rightMouthY), - ]; - const face: FaceDetection = { - box, - landmarks, - probability, - // detectionMethod: this.method, - }; - faces.push(face); - } - return faces; - } - - public getRelativeDetection( - faceDetection: FaceDetection, - dimensions: Dimensions, - ): FaceDetection { - const oldBox: Box = faceDetection.box; - const box = new Box({ - x: oldBox.x / dimensions.width, - y: oldBox.y / dimensions.height, - width: oldBox.width / dimensions.width, - height: oldBox.height / dimensions.height, - }); - const oldLandmarks: Point[] = faceDetection.landmarks; - const landmarks = oldLandmarks.map((l) => { - return new Point(l.x / dimensions.width, l.y / dimensions.height); - }); +/** + * Detect faces in the given {@link imageBitmap}. + * + * The ML model used is YOLO, running in an ONNX runtime. + */ +export const detectFaces = async ( + imageBitmap: ImageBitmap, +): Promise> => { + const maxFaceDistancePercent = Math.sqrt(2) / 100; + const maxFaceDistance = imageBitmap.width * maxFaceDistancePercent; + const preprocessResult = preprocessImageBitmapToFloat32ChannelsFirst( + imageBitmap, + 640, + 640, + ); + const data = preprocessResult.data; + const resized = preprocessResult.newSize; + const outputData = await workerBridge.detectFaces(data); + const faces = getFacesFromYoloOutput(outputData as Float32Array, 0.7); + const inBox = newBox(0, 0, resized.width, resized.height); + const toBox = newBox(0, 0, imageBitmap.width, imageBitmap.height); + const transform = computeTransformToBox(inBox, toBox); + const faceDetections: Array = faces?.map((f) => { + const box = transformBox(f.box, transform); + const normLandmarks = f.landmarks; + const landmarks = transformPoints(normLandmarks, transform); return { box, landmarks, - probability: faceDetection.probability, - }; - } -} + probability: f.probability as number, + } as FaceDetection; + }); + return removeDuplicateDetections(faceDetections, maxFaceDistance); +}; -export default new YoloFaceDetectionService(); +const preprocessImageBitmapToFloat32ChannelsFirst = ( + imageBitmap: ImageBitmap, + requiredWidth: number, + requiredHeight: number, + maintainAspectRatio: boolean = true, + normFunction: (pixelValue: number) => number = normalizePixelBetween0And1, +) => { + // Create an OffscreenCanvas and set its size + const offscreenCanvas = new OffscreenCanvas( + imageBitmap.width, + imageBitmap.height, + ); + const ctx = offscreenCanvas.getContext("2d"); + ctx.drawImage(imageBitmap, 0, 0, imageBitmap.width, imageBitmap.height); + const imageData = ctx.getImageData( + 0, + 0, + imageBitmap.width, + imageBitmap.height, + ); + const pixelData = imageData.data; + + let scaleW = requiredWidth / imageBitmap.width; + let scaleH = requiredHeight / imageBitmap.height; + if (maintainAspectRatio) { + const scale = Math.min( + requiredWidth / imageBitmap.width, + requiredHeight / imageBitmap.height, + ); + scaleW = scale; + scaleH = scale; + } + const scaledWidth = clamp( + Math.round(imageBitmap.width * scaleW), + 0, + requiredWidth, + ); + const scaledHeight = clamp( + Math.round(imageBitmap.height * scaleH), + 0, + requiredHeight, + ); + + const processedImage = new Float32Array( + 1 * 3 * requiredWidth * requiredHeight, + ); + + // Populate the Float32Array with normalized pixel values + let pixelIndex = 0; + const channelOffsetGreen = requiredHeight * requiredWidth; + const channelOffsetBlue = 2 * requiredHeight * requiredWidth; + for (let h = 0; h < requiredHeight; h++) { + for (let w = 0; w < requiredWidth; w++) { + let pixel: { + r: number; + g: number; + b: number; + }; + if (w >= scaledWidth || h >= scaledHeight) { + pixel = { r: 114, g: 114, b: 114 }; + } else { + pixel = getPixelBilinear( + w / scaleW, + h / scaleH, + pixelData, + imageBitmap.width, + imageBitmap.height, + ); + } + processedImage[pixelIndex] = normFunction(pixel.r); + processedImage[pixelIndex + channelOffsetGreen] = normFunction( + pixel.g, + ); + processedImage[pixelIndex + channelOffsetBlue] = normFunction( + pixel.b, + ); + pixelIndex++; + } + } + + return { + data: processedImage, + originalSize: { + width: imageBitmap.width, + height: imageBitmap.height, + }, + newSize: { width: scaledWidth, height: scaledHeight }, + }; +}; + +// The rowOutput is a Float32Array of shape [25200, 16], where each row represents a bounding box. +const getFacesFromYoloOutput = ( + rowOutput: Float32Array, + minScore: number, +): Array => { + const faces: Array = []; + // iterate over each row + for (let i = 0; i < rowOutput.length; i += 16) { + const score = rowOutput[i + 4]; + if (score < minScore) { + continue; + } + // The first 4 values represent the bounding box's coordinates (x1, y1, x2, y2) + const xCenter = rowOutput[i]; + const yCenter = rowOutput[i + 1]; + const width = rowOutput[i + 2]; + const height = rowOutput[i + 3]; + const xMin = xCenter - width / 2.0; // topLeft + const yMin = yCenter - height / 2.0; // topLeft + + const leftEyeX = rowOutput[i + 5]; + const leftEyeY = rowOutput[i + 6]; + const rightEyeX = rowOutput[i + 7]; + const rightEyeY = rowOutput[i + 8]; + const noseX = rowOutput[i + 9]; + const noseY = rowOutput[i + 10]; + const leftMouthX = rowOutput[i + 11]; + const leftMouthY = rowOutput[i + 12]; + const rightMouthX = rowOutput[i + 13]; + const rightMouthY = rowOutput[i + 14]; + + const box = new Box({ + x: xMin, + y: yMin, + width: width, + height: height, + }); + const probability = score as number; + const landmarks = [ + new Point(leftEyeX, leftEyeY), + new Point(rightEyeX, rightEyeY), + new Point(noseX, noseY), + new Point(leftMouthX, leftMouthY), + new Point(rightMouthX, rightMouthY), + ]; + const face: FaceDetection = { + box, + landmarks, + probability, + // detectionMethod: this.method, + }; + faces.push(face); + } + return faces; +}; + +export const getRelativeDetection = ( + faceDetection: FaceDetection, + dimensions: Dimensions, +): FaceDetection => { + const oldBox: Box = faceDetection.box; + const box = new Box({ + x: oldBox.x / dimensions.width, + y: oldBox.y / dimensions.height, + width: oldBox.width / dimensions.width, + height: oldBox.height / dimensions.height, + }); + const oldLandmarks: Point[] = faceDetection.landmarks; + const landmarks = oldLandmarks.map((l) => { + return new Point(l.x / dimensions.width, l.y / dimensions.height); + }); + return { + box, + landmarks, + probability: faceDetection.probability, + }; +}; /** * Removes duplicate face detections from an array of detections. diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index e6321cb76a..d09380791e 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -198,7 +198,6 @@ export interface MLSyncContext { token: string; userID: number; - faceDetectionService: FaceDetectionService; faceCropService: FaceCropService; faceEmbeddingService: FaceEmbeddingService; @@ -240,16 +239,6 @@ export interface MLLibraryData { export declare type MLIndex = "files" | "people"; -export interface FaceDetectionService { - method: Versioned; - - detectFaces(image: ImageBitmap): Promise>; - getRelativeDetection( - faceDetection: FaceDetection, - imageDimensions: Dimensions, - ): FaceDetection; -} - export interface FaceCropService { method: Versioned; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 843d2106ab..ecf80132c9 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -2,6 +2,7 @@ import { openCache } from "@/next/blob-cache"; import log from "@/next/log"; import { faceAlignment } from "services/face/align"; import mlIDbStorage from "services/face/db"; +import { detectFaces, getRelativeDetection } from "services/face/detect-face"; import { DetectedFace, Face, @@ -10,9 +11,9 @@ import { type FaceAlignment, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; -import { detectBlur } from "../face/detect-blur"; import { clusterFaces } from "../face/cluster"; import { getFaceCrop } from "../face/crop"; +import { detectBlur } from "../face/detect-blur"; import { fetchImageBitmap, fetchImageBitmapForContext, @@ -26,11 +27,13 @@ class FaceService { fileContext: MLSyncFileContext, ) { const { newMlFile } = fileContext; - newMlFile.faceDetectionMethod = syncContext.faceDetectionService.method; + newMlFile.faceDetectionMethod = { + value: "YoloFace", + version: 1, + }; fileContext.newDetection = true; const imageBitmap = await fetchImageBitmapForContext(fileContext); - const faceDetections = - await syncContext.faceDetectionService.detectFaces(imageBitmap); + const faceDetections = await detectFaces(imageBitmap); // TODO: reenable faces filtering based on width const detectedFaces = faceDetections?.map((detection) => { return { @@ -123,11 +126,10 @@ class FaceService { for (let i = 0; i < newMlFile.faces.length; i++) { const face = newMlFile.faces[i]; if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative - face.detection = - syncContext.faceDetectionService.getRelativeDetection( - face.detection, - newMlFile.imageDimensions, - ); + face.detection = getRelativeDetection( + face.detection, + newMlFile.imageDimensions, + ); } } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 72c34694ae..6cb552864c 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -15,8 +15,6 @@ import { Face, FaceCropService, FaceDetection, - FaceDetectionMethod, - FaceDetectionService, FaceEmbeddingMethod, FaceEmbeddingService, Landmark, @@ -31,12 +29,10 @@ import { import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import FaceService from "./faceService"; -import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; - import { fetchImageBitmapForContext } from "../face/image"; import { syncPeopleIndex } from "../face/people"; -import yoloFaceDetectionService from "../face/detect-face"; +import FaceService from "./faceService"; +import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; /** * TODO-ML(MR): What and why. @@ -110,16 +106,6 @@ export async function updateMLSearchConfig(newConfig: MLSearchConfig) { } export class MLFactory { - public static getFaceDetectionService( - method: FaceDetectionMethod, - ): FaceDetectionService { - if (method === "YoloFace") { - return yoloFaceDetectionService; - } - - throw Error("Unknon face detection method: " + method); - } - public static getFaceEmbeddingService( method: FaceEmbeddingMethod, ): FaceEmbeddingService { @@ -135,7 +121,6 @@ export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; - public faceDetectionService: FaceDetectionService; public faceCropService: FaceCropService; public faceEmbeddingService: FaceEmbeddingService; @@ -163,8 +148,6 @@ export class LocalMLSyncContext implements MLSyncContext { this.token = token; this.userID = userID; - this.faceDetectionService = - MLFactory.getFaceDetectionService("YoloFace"); this.faceEmbeddingService = MLFactory.getFaceEmbeddingService("MobileFaceNet"); From 48cc9a08b6f5c3a42307391c95b4362426f9fd25 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:29:45 +0530 Subject: [PATCH 31/47] Rename --- web/apps/photos/src/services/face/{detect-blur.ts => blur.ts} | 0 .../photos/src/services/face/{detect-face.ts => detect.ts} | 0 web/apps/photos/src/services/machineLearning/faceService.ts | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename web/apps/photos/src/services/face/{detect-blur.ts => blur.ts} (100%) rename web/apps/photos/src/services/face/{detect-face.ts => detect.ts} (100%) diff --git a/web/apps/photos/src/services/face/detect-blur.ts b/web/apps/photos/src/services/face/blur.ts similarity index 100% rename from web/apps/photos/src/services/face/detect-blur.ts rename to web/apps/photos/src/services/face/blur.ts diff --git a/web/apps/photos/src/services/face/detect-face.ts b/web/apps/photos/src/services/face/detect.ts similarity index 100% rename from web/apps/photos/src/services/face/detect-face.ts rename to web/apps/photos/src/services/face/detect.ts diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index ecf80132c9..c358cc2916 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -2,7 +2,7 @@ import { openCache } from "@/next/blob-cache"; import log from "@/next/log"; import { faceAlignment } from "services/face/align"; import mlIDbStorage from "services/face/db"; -import { detectFaces, getRelativeDetection } from "services/face/detect-face"; +import { detectFaces, getRelativeDetection } from "services/face/detect"; import { DetectedFace, Face, @@ -13,7 +13,7 @@ import { import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { clusterFaces } from "../face/cluster"; import { getFaceCrop } from "../face/crop"; -import { detectBlur } from "../face/detect-blur"; +import { detectBlur } from "../face/blur"; import { fetchImageBitmap, fetchImageBitmapForContext, From 73946d9b8ef6a4a5ef4dce361948eb9bdfd51fab Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:36:03 +0530 Subject: [PATCH 32/47] Tinker --- web/apps/photos/src/services/face/detect.ts | 57 +++++++++++---------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/web/apps/photos/src/services/face/detect.ts b/web/apps/photos/src/services/face/detect.ts index 5c6b633885..a6ccf77c41 100644 --- a/web/apps/photos/src/services/face/detect.ts +++ b/web/apps/photos/src/services/face/detect.ts @@ -39,7 +39,7 @@ export const detectFaces = async ( const data = preprocessResult.data; const resized = preprocessResult.newSize; const outputData = await workerBridge.detectFaces(data); - const faces = getFacesFromYoloOutput(outputData as Float32Array, 0.7); + const faces = getFacesFromYOLOOutput(outputData as Float32Array, 0.7); const inBox = newBox(0, 0, resized.width, resized.height); const toBox = newBox(0, 0, imageBitmap.width, imageBitmap.height); const transform = computeTransformToBox(inBox, toBox); @@ -63,7 +63,7 @@ const preprocessImageBitmapToFloat32ChannelsFirst = ( maintainAspectRatio: boolean = true, normFunction: (pixelValue: number) => number = normalizePixelBetween0And1, ) => { - // Create an OffscreenCanvas and set its size + // Create an OffscreenCanvas and set its size. const offscreenCanvas = new OffscreenCanvas( imageBitmap.width, imageBitmap.height, @@ -146,19 +146,25 @@ const preprocessImageBitmapToFloat32ChannelsFirst = ( }; }; -// The rowOutput is a Float32Array of shape [25200, 16], where each row represents a bounding box. -const getFacesFromYoloOutput = ( +/** + * @param rowOutput A Float32Array of shape [25200, 16], where each row + * represents a bounding box. + */ +const getFacesFromYOLOOutput = ( rowOutput: Float32Array, minScore: number, ): Array => { const faces: Array = []; - // iterate over each row + // Iterate over each row. for (let i = 0; i < rowOutput.length; i += 16) { const score = rowOutput[i + 4]; if (score < minScore) { continue; } - // The first 4 values represent the bounding box's coordinates (x1, y1, x2, y2) + // The first 4 values represent the bounding box's coordinates: + // + // (x1, y1, x2, y2) + // const xCenter = rowOutput[i]; const yCenter = rowOutput[i + 1]; const width = rowOutput[i + 2]; @@ -191,13 +197,7 @@ const getFacesFromYoloOutput = ( new Point(leftMouthX, leftMouthY), new Point(rightMouthX, rightMouthY), ]; - const face: FaceDetection = { - box, - landmarks, - probability, - // detectionMethod: this.method, - }; - faces.push(face); + faces.push({ box, landmarks, probability }); } return faces; }; @@ -217,30 +217,34 @@ export const getRelativeDetection = ( const landmarks = oldLandmarks.map((l) => { return new Point(l.x / dimensions.width, l.y / dimensions.height); }); - return { - box, - landmarks, - probability: faceDetection.probability, - }; + const probability = faceDetection.probability; + return { box, landmarks, probability }; }; /** * Removes duplicate face detections from an array of detections. * - * This function sorts the detections by their probability in descending order, then iterates over them. - * For each detection, it calculates the Euclidean distance to all other detections. - * If the distance is less than or equal to the specified threshold (`withinDistance`), the other detection is considered a duplicate and is removed. + * This function sorts the detections by their probability in descending order, + * then iterates over them. + * + * For each detection, it calculates the Euclidean distance to all other + * detections. + * + * If the distance is less than or equal to the specified threshold + * (`withinDistance`), the other detection is considered a duplicate and is + * removed. * * @param detections - An array of face detections to remove duplicates from. - * @param withinDistance - The maximum Euclidean distance between two detections for them to be considered duplicates. + * + * @param withinDistance - The maximum Euclidean distance between two detections + * for them to be considered duplicates. * * @returns An array of face detections with duplicates removed. */ -function removeDuplicateDetections( +const removeDuplicateDetections = ( detections: Array, withinDistance: number, -) { - // console.time('removeDuplicates'); +) => { detections.sort((a, b) => b.probability - a.probability); const isSelected = new Map(); for (let i = 0; i < detections.length; i++) { @@ -268,9 +272,8 @@ function removeDuplicateDetections( for (let i = 0; i < detections.length; i++) { isSelected.get(i) && uniques.push(detections[i]); } - // console.timeEnd('removeDuplicates'); return uniques; -} +}; function getDetectionCenter(detection: FaceDetection) { const center = new Point(0, 0); From 43a3df5bbfd796fffcc8fb6533941c43313c6e11 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:41:50 +0530 Subject: [PATCH 33/47] embeddings --- desktop/src/main/ipc.ts | 6 +-- desktop/src/main/services/ml-face.ts | 2 +- desktop/src/preload.ts | 6 +-- web/apps/photos/src/services/face/blur.ts | 2 +- web/apps/photos/src/services/face/detect.ts | 2 +- .../src/services/machineLearning/embed.ts | 26 ++++++++++++ .../machineLearning/machineLearningService.ts | 2 +- .../mobileFaceNetEmbeddingService.ts | 41 ------------------- web/packages/next/types/ipc.ts | 4 +- web/packages/next/worker/comlink-worker.ts | 4 +- 10 files changed, 40 insertions(+), 55 deletions(-) create mode 100644 web/apps/photos/src/services/machineLearning/embed.ts delete mode 100644 web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index ab5af51a1c..e74d5e9d2e 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -46,7 +46,7 @@ import { clipImageEmbedding, clipTextEmbeddingIfAvailable, } from "./services/ml-clip"; -import { detectFaces, faceEmbedding } from "./services/ml-face"; +import { detectFaces, faceEmbeddings } from "./services/ml-face"; import { encryptionKey, saveEncryptionKey } from "./services/store"; import { clearPendingUploads, @@ -182,8 +182,8 @@ export const attachIPCHandlers = () => { detectFaces(input), ); - ipcMain.handle("faceEmbedding", (_, input: Float32Array) => - faceEmbedding(input), + ipcMain.handle("faceEmbeddings", (_, input: Float32Array) => + faceEmbeddings(input), ); ipcMain.handle("legacyFaceCrop", (_, faceID: string) => diff --git a/desktop/src/main/services/ml-face.ts b/desktop/src/main/services/ml-face.ts index 9765252555..33157694fc 100644 --- a/desktop/src/main/services/ml-face.ts +++ b/desktop/src/main/services/ml-face.ts @@ -32,7 +32,7 @@ const cachedFaceEmbeddingSession = makeCachedInferenceSession( 5286998 /* 5 MB */, ); -export const faceEmbedding = async (input: Float32Array) => { +export const faceEmbeddings = async (input: Float32Array) => { // Dimension of each face (alias) const mobileFaceNetFaceSize = 112; // Smaller alias diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 764609193c..c5a1d0d317 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -162,8 +162,8 @@ const clipTextEmbeddingIfAvailable = (text: string) => const detectFaces = (input: Float32Array) => ipcRenderer.invoke("detectFaces", input); -const faceEmbedding = (input: Float32Array) => - ipcRenderer.invoke("faceEmbedding", input); +const faceEmbeddings = (input: Float32Array) => + ipcRenderer.invoke("faceEmbeddings", input); const legacyFaceCrop = (faceID: string) => ipcRenderer.invoke("legacyFaceCrop", faceID); @@ -343,7 +343,7 @@ contextBridge.exposeInMainWorld("electron", { clipImageEmbedding, clipTextEmbeddingIfAvailable, detectFaces, - faceEmbedding, + faceEmbeddings, legacyFaceCrop, // - Watch diff --git a/web/apps/photos/src/services/face/blur.ts b/web/apps/photos/src/services/face/blur.ts index 2c6083d6e0..71420b1dfc 100644 --- a/web/apps/photos/src/services/face/blur.ts +++ b/web/apps/photos/src/services/face/blur.ts @@ -1,6 +1,6 @@ import { Face } from "services/face/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; -import { mobileFaceNetFaceSize } from "../machineLearning/mobileFaceNetEmbeddingService"; +import { mobileFaceNetFaceSize } from "../machineLearning/embed"; /** * Laplacian blur detection. diff --git a/web/apps/photos/src/services/face/detect.ts b/web/apps/photos/src/services/face/detect.ts index a6ccf77c41..39b8430628 100644 --- a/web/apps/photos/src/services/face/detect.ts +++ b/web/apps/photos/src/services/face/detect.ts @@ -24,7 +24,7 @@ import { /** * Detect faces in the given {@link imageBitmap}. * - * The ML model used is YOLO, running in an ONNX runtime. + * The model used is YOLO, running in an ONNX runtime. */ export const detectFaces = async ( imageBitmap: ImageBitmap, diff --git a/web/apps/photos/src/services/machineLearning/embed.ts b/web/apps/photos/src/services/machineLearning/embed.ts new file mode 100644 index 0000000000..5dfbf70615 --- /dev/null +++ b/web/apps/photos/src/services/machineLearning/embed.ts @@ -0,0 +1,26 @@ +import { workerBridge } from "@/next/worker/worker-bridge"; +import { FaceEmbedding } from "services/face/types"; + +export const mobileFaceNetFaceSize = 112; + +/** + * Compute embeddings for the given {@link faceData}. + * + * The model used is MobileFaceNet, running in an ONNX runtime. + */ +export const getFaceEmbeddings = async ( + faceData: Float32Array, +): Promise> => { + const outputData = await workerBridge.faceEmbeddings(faceData); + + const embeddingSize = 192; + const embeddings = new Array( + outputData.length / embeddingSize, + ); + for (let i = 0; i < embeddings.length; i++) { + embeddings[i] = new Float32Array( + outputData.slice(i * embeddingSize, (i + 1) * embeddingSize), + ); + } + return embeddings; +}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 6cb552864c..ea75074a38 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -31,8 +31,8 @@ import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { fetchImageBitmapForContext } from "../face/image"; import { syncPeopleIndex } from "../face/people"; +import mobileFaceNetEmbeddingService from "./embed"; import FaceService from "./faceService"; -import mobileFaceNetEmbeddingService from "./mobileFaceNetEmbeddingService"; /** * TODO-ML(MR): What and why. diff --git a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts b/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts deleted file mode 100644 index 46288d1699..0000000000 --- a/web/apps/photos/src/services/machineLearning/mobileFaceNetEmbeddingService.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { workerBridge } from "@/next/worker/worker-bridge"; -import { - FaceEmbedding, - FaceEmbeddingMethod, - FaceEmbeddingService, - Versioned, -} from "services/face/types"; - -export const mobileFaceNetFaceSize = 112; - -class MobileFaceNetEmbeddingService implements FaceEmbeddingService { - public method: Versioned; - public faceSize: number; - - public constructor() { - this.method = { - value: "MobileFaceNet", - version: 2, - }; - this.faceSize = mobileFaceNetFaceSize; - } - - public async getFaceEmbeddings( - faceData: Float32Array, - ): Promise> { - const outputData = await workerBridge.faceEmbedding(faceData); - - const embeddingSize = 192; - const embeddings = new Array( - outputData.length / embeddingSize, - ); - for (let i = 0; i < embeddings.length; i++) { - embeddings[i] = new Float32Array( - outputData.slice(i * embeddingSize, (i + 1) * embeddingSize), - ); - } - return embeddings; - } -} - -export default new MobileFaceNetEmbeddingService(); diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index f468f9ab37..7d5866cdb1 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -332,12 +332,12 @@ export interface Electron { detectFaces: (input: Float32Array) => Promise; /** - * Return a MobileFaceNet embedding for the given face data. + * Return a MobileFaceNet embeddings for the given faces. * * Both the input and output are opaque binary data whose internal structure * is specific to our implementation and the model (MobileFaceNet) we use. */ - faceEmbedding: (input: Float32Array) => Promise; + faceEmbeddings: (input: Float32Array) => Promise; /** * Return a face crop stored by a previous version of ML. diff --git a/web/packages/next/worker/comlink-worker.ts b/web/packages/next/worker/comlink-worker.ts index 5929e5361b..cb90d85f8f 100644 --- a/web/packages/next/worker/comlink-worker.ts +++ b/web/packages/next/worker/comlink-worker.ts @@ -47,8 +47,8 @@ const workerBridge = { convertToJPEG: (imageData: Uint8Array) => ensureElectron().convertToJPEG(imageData), detectFaces: (input: Float32Array) => ensureElectron().detectFaces(input), - faceEmbedding: (input: Float32Array) => - ensureElectron().faceEmbedding(input), + faceEmbeddings: (input: Float32Array) => + ensureElectron().faceEmbeddings(input), }; export type WorkerBridge = typeof workerBridge; From 84c737ddd3e0b750b78bea678e22e8a07bb1d428 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:50:16 +0530 Subject: [PATCH 34/47] Unclass --- web/apps/photos/src/services/face/blur.ts | 2 +- .../{machineLearning => face}/embed.ts | 2 +- web/apps/photos/src/services/face/types.ts | 8 -------- .../services/machineLearning/faceService.ts | 15 ++++++++------- .../machineLearning/machineLearningService.ts | 19 ------------------- 5 files changed, 10 insertions(+), 36 deletions(-) rename web/apps/photos/src/services/{machineLearning => face}/embed.ts (94%) diff --git a/web/apps/photos/src/services/face/blur.ts b/web/apps/photos/src/services/face/blur.ts index 71420b1dfc..c790812974 100644 --- a/web/apps/photos/src/services/face/blur.ts +++ b/web/apps/photos/src/services/face/blur.ts @@ -1,6 +1,6 @@ import { Face } from "services/face/types"; import { createGrayscaleIntMatrixFromNormalized2List } from "utils/image"; -import { mobileFaceNetFaceSize } from "../machineLearning/embed"; +import { mobileFaceNetFaceSize } from "./embed"; /** * Laplacian blur detection. diff --git a/web/apps/photos/src/services/machineLearning/embed.ts b/web/apps/photos/src/services/face/embed.ts similarity index 94% rename from web/apps/photos/src/services/machineLearning/embed.ts rename to web/apps/photos/src/services/face/embed.ts index 5dfbf70615..2e0977ea1c 100644 --- a/web/apps/photos/src/services/machineLearning/embed.ts +++ b/web/apps/photos/src/services/face/embed.ts @@ -8,7 +8,7 @@ export const mobileFaceNetFaceSize = 112; * * The model used is MobileFaceNet, running in an ONNX runtime. */ -export const getFaceEmbeddings = async ( +export const faceEmbeddings = async ( faceData: Float32Array, ): Promise> => { const outputData = await workerBridge.faceEmbeddings(faceData); diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index d09380791e..44b3321f69 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -199,7 +199,6 @@ export interface MLSyncContext { userID: number; faceCropService: FaceCropService; - faceEmbeddingService: FaceEmbeddingService; localFilesMap: Map; outOfSyncFiles: EnteFile[]; @@ -248,13 +247,6 @@ export interface FaceCropService { ): Promise; } -export interface FaceEmbeddingService { - method: Versioned; - faceSize: number; - - getFaceEmbeddings(faceImages: Float32Array): Promise>; -} - export interface MachineLearningWorker { closeLocalSyncContext(): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index c358cc2916..80bda398aa 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -3,6 +3,7 @@ import log from "@/next/log"; import { faceAlignment } from "services/face/align"; import mlIDbStorage from "services/face/db"; import { detectFaces, getRelativeDetection } from "services/face/detect"; +import { faceEmbeddings, mobileFaceNetFaceSize } from "services/face/embed"; import { DetectedFace, Face, @@ -11,9 +12,9 @@ import { type FaceAlignment, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { detectBlur } from "../face/blur"; import { clusterFaces } from "../face/cluster"; import { getFaceCrop } from "../face/crop"; -import { detectBlur } from "../face/blur"; import { fetchImageBitmap, fetchImageBitmapForContext, @@ -86,7 +87,7 @@ class FaceService { const faceAlignments = newMlFile.faces.map((f) => f.alignment); const faceImages = await extractFaceImagesToFloat32( faceAlignments, - syncContext.faceEmbeddingService.faceSize, + mobileFaceNetFaceSize, imageBitmap, ); const blurValues = detectBlur(faceImages, newMlFile.faces); @@ -104,15 +105,15 @@ class FaceService { alignedFacesInput: Float32Array, ) { const { newMlFile } = fileContext; - newMlFile.faceEmbeddingMethod = syncContext.faceEmbeddingService.method; + newMlFile.faceEmbeddingMethod = { + value: "MobileFaceNet", + version: 2, + }; // TODO: when not storing face crops, image will be needed to extract faces // fileContext.imageBitmap || // (await this.getImageBitmap(fileContext)); - const embeddings = - await syncContext.faceEmbeddingService.getFaceEmbeddings( - alignedFacesInput, - ); + const embeddings = await faceEmbeddings(alignedFacesInput); newMlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); log.info("[MLService] facesWithEmbeddings: ", newMlFile.faces.length); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index ea75074a38..8e5c7b103b 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -15,8 +15,6 @@ import { Face, FaceCropService, FaceDetection, - FaceEmbeddingMethod, - FaceEmbeddingService, Landmark, MLLibraryData, MLSearchConfig, @@ -31,7 +29,6 @@ import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { fetchImageBitmapForContext } from "../face/image"; import { syncPeopleIndex } from "../face/people"; -import mobileFaceNetEmbeddingService from "./embed"; import FaceService from "./faceService"; /** @@ -105,24 +102,11 @@ export async function updateMLSearchConfig(newConfig: MLSearchConfig) { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); } -export class MLFactory { - public static getFaceEmbeddingService( - method: FaceEmbeddingMethod, - ): FaceEmbeddingService { - if (method === "MobileFaceNet") { - return mobileFaceNetEmbeddingService; - } - - throw Error("Unknon face embedding method: " + method); - } -} - export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; public faceCropService: FaceCropService; - public faceEmbeddingService: FaceEmbeddingService; public localFilesMap: Map; public outOfSyncFiles: EnteFile[]; @@ -148,9 +132,6 @@ export class LocalMLSyncContext implements MLSyncContext { this.token = token; this.userID = userID; - this.faceEmbeddingService = - MLFactory.getFaceEmbeddingService("MobileFaceNet"); - this.outOfSyncFiles = []; this.nSyncedFiles = 0; this.nSyncedFaces = 0; From 3db91d20349006b1e94274266ae1b41eece55226 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 13:52:45 +0530 Subject: [PATCH 35/47] Cleanup --- web/apps/photos/src/services/face/types.ts | 11 ----------- .../src/services/machineLearning/faceService.ts | 5 ++++- .../machineLearning/machineLearningService.ts | 3 --- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index 44b3321f69..40aa29aee8 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -198,8 +198,6 @@ export interface MLSyncContext { token: string; userID: number; - faceCropService: FaceCropService; - localFilesMap: Map; outOfSyncFiles: EnteFile[]; nSyncedFiles: number; @@ -238,15 +236,6 @@ export interface MLLibraryData { export declare type MLIndex = "files" | "people"; -export interface FaceCropService { - method: Versioned; - - getFaceCrop( - imageBitmap: ImageBitmap, - face: FaceDetection, - ): Promise; -} - export interface MachineLearningWorker { closeLocalSyncContext(): Promise; diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts index 80bda398aa..74d7f7f246 100644 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ b/web/apps/photos/src/services/machineLearning/faceService.ts @@ -58,7 +58,10 @@ class FaceService { ) { const { newMlFile } = fileContext; const imageBitmap = await fetchImageBitmapForContext(fileContext); - newMlFile.faceCropMethod = syncContext.faceCropService.method; + newMlFile.faceCropMethod = { + value: "ArcFace", + version: 1, + }; for (const face of newMlFile.faces) { await this.saveFaceCrop(imageBitmap, face); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 8e5c7b103b..9ff1e1e249 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -13,7 +13,6 @@ import { putEmbedding } from "services/embeddingService"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { Face, - FaceCropService, FaceDetection, Landmark, MLLibraryData, @@ -106,8 +105,6 @@ export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; - public faceCropService: FaceCropService; - public localFilesMap: Map; public outOfSyncFiles: EnteFile[]; public nSyncedFiles: number; From 054b4c7cfb382914f23f7883878fdbc3eb252ff4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:15:45 +0530 Subject: [PATCH 36/47] Unclass indexer --- web/apps/photos/src/services/face/f-index.ts | 260 ++++++++++++++++++ web/apps/photos/src/services/face/people.ts | 12 +- .../services/machineLearning/faceService.ts | 233 ---------------- .../machineLearning/machineLearningService.ts | 42 +-- 4 files changed, 271 insertions(+), 276 deletions(-) create mode 100644 web/apps/photos/src/services/face/f-index.ts delete mode 100644 web/apps/photos/src/services/machineLearning/faceService.ts diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts new file mode 100644 index 0000000000..9f282cdced --- /dev/null +++ b/web/apps/photos/src/services/face/f-index.ts @@ -0,0 +1,260 @@ +import { openCache } from "@/next/blob-cache"; +import log from "@/next/log"; +import { faceAlignment } from "services/face/align"; +import mlIDbStorage from "services/face/db"; +import { detectFaces, getRelativeDetection } from "services/face/detect"; +import { faceEmbeddings, mobileFaceNetFaceSize } from "services/face/embed"; +import { + DetectedFace, + Face, + MLSyncContext, + MLSyncFileContext, + type FaceAlignment, +} from "services/face/types"; +import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; +import { detectBlur } from "./blur"; +import { clusterFaces } from "./cluster"; +import { getFaceCrop } from "./crop"; +import { + fetchImageBitmap, + fetchImageBitmapForContext, + getFaceId, + getLocalFile, +} from "./image"; + +export const syncFileAnalyzeFaces = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, +) => { + const { newMlFile } = fileContext; + const startTime = Date.now(); + + await syncFileFaceDetections(syncContext, fileContext); + + if (newMlFile.faces && newMlFile.faces.length > 0) { + await syncFileFaceCrops(syncContext, fileContext); + + const alignedFacesData = await syncFileFaceAlignments( + syncContext, + fileContext, + ); + + await syncFileFaceEmbeddings( + syncContext, + fileContext, + alignedFacesData, + ); + + await syncFileFaceMakeRelativeDetections(syncContext, fileContext); + } + log.debug( + () => + `Face detection for file ${fileContext.enteFile.id} took ${Math.round(Date.now() - startTime)} ms`, + ); +}; + +const syncFileFaceDetections = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, +) => { + const { newMlFile } = fileContext; + newMlFile.faceDetectionMethod = { + value: "YoloFace", + version: 1, + }; + fileContext.newDetection = true; + const imageBitmap = await fetchImageBitmapForContext(fileContext); + const faceDetections = await detectFaces(imageBitmap); + // TODO: reenable faces filtering based on width + const detectedFaces = faceDetections?.map((detection) => { + return { + fileId: fileContext.enteFile.id, + detection, + } as DetectedFace; + }); + newMlFile.faces = detectedFaces?.map((detectedFace) => ({ + ...detectedFace, + id: getFaceId(detectedFace, newMlFile.imageDimensions), + })); + // ?.filter((f) => + // f.box.width > syncContext.config.faceDetection.minFaceSize + // ); + log.info("[MLService] Detected Faces: ", newMlFile.faces?.length); +}; + +const syncFileFaceCrops = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, +) => { + const { newMlFile } = fileContext; + const imageBitmap = await fetchImageBitmapForContext(fileContext); + newMlFile.faceCropMethod = { + value: "ArcFace", + version: 1, + }; + + for (const face of newMlFile.faces) { + await saveFaceCrop(imageBitmap, face); + } +}; + +const syncFileFaceAlignments = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, +): Promise => { + const { newMlFile } = fileContext; + newMlFile.faceAlignmentMethod = { + value: "ArcFace", + version: 1, + }; + fileContext.newAlignment = true; + const imageBitmap = + fileContext.imageBitmap || + (await fetchImageBitmapForContext(fileContext)); + + // Execute the face alignment calculations + for (const face of newMlFile.faces) { + face.alignment = faceAlignment(face.detection); + } + // Extract face images and convert to Float32Array + const faceAlignments = newMlFile.faces.map((f) => f.alignment); + const faceImages = await extractFaceImagesToFloat32( + faceAlignments, + mobileFaceNetFaceSize, + imageBitmap, + ); + const blurValues = detectBlur(faceImages, newMlFile.faces); + newMlFile.faces.forEach((f, i) => (f.blurValue = blurValues[i])); + + imageBitmap.close(); + log.info("[MLService] alignedFaces: ", newMlFile.faces?.length); + + return faceImages; +}; + +const syncFileFaceEmbeddings = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, + alignedFacesInput: Float32Array, +) => { + const { newMlFile } = fileContext; + newMlFile.faceEmbeddingMethod = { + value: "MobileFaceNet", + version: 2, + }; + // TODO: when not storing face crops, image will be needed to extract faces + // fileContext.imageBitmap || + // (await this.getImageBitmap(fileContext)); + + const embeddings = await faceEmbeddings(alignedFacesInput); + newMlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); + + log.info("[MLService] facesWithEmbeddings: ", newMlFile.faces.length); +}; + +const syncFileFaceMakeRelativeDetections = async ( + syncContext: MLSyncContext, + fileContext: MLSyncFileContext, +) => { + const { newMlFile } = fileContext; + for (let i = 0; i < newMlFile.faces.length; i++) { + const face = newMlFile.faces[i]; + if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative + face.detection = getRelativeDetection( + face.detection, + newMlFile.imageDimensions, + ); + } +}; + +export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { + const faceCrop = getFaceCrop(imageBitmap, face.detection); + + const blob = await imageBitmapToBlob(faceCrop.image); + + const cache = await openCache("face-crops"); + await cache.put(face.id, blob); + + faceCrop.image.close(); + + return blob; +}; + +export const getAllSyncedFacesMap = async (syncContext: MLSyncContext) => { + if (syncContext.allSyncedFacesMap) { + return syncContext.allSyncedFacesMap; + } + + syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap(); + return syncContext.allSyncedFacesMap; +}; + +export const runFaceClustering = async ( + syncContext: MLSyncContext, + allFaces: Array, +) => { + // await this.init(); + + if (!allFaces || allFaces.length < 50) { + log.info( + `Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`, + ); + return; + } + + log.info("Running clustering allFaces: ", allFaces.length); + syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( + allFaces.map((f) => Array.from(f.embedding)), + ); + syncContext.mlLibraryData.faceClusteringMethod = { + value: "Hdbscan", + version: 1, + }; + log.info( + "[MLService] Got face clustering results: ", + JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), + ); + + // syncContext.faceClustersWithNoise = { + // clusters: syncContext.faceClusteringResults.clusters.map( + // (faces) => ({ + // faces, + // }) + // ), + // noise: syncContext.faceClusteringResults.noise, + // }; +}; + +export const regenerateFaceCrop = async (faceID: string) => { + const fileID = Number(faceID.split("-")[0]); + const personFace = await mlIDbStorage.getFace(fileID, faceID); + if (!personFace) { + throw Error("Face not found"); + } + + const file = await getLocalFile(personFace.fileId); + const imageBitmap = await fetchImageBitmap(file); + return await saveFaceCrop(imageBitmap, personFace); +}; + +async function extractFaceImagesToFloat32( + faceAlignments: Array, + faceSize: number, + image: ImageBitmap, +): Promise { + const faceData = new Float32Array( + faceAlignments.length * faceSize * faceSize * 3, + ); + for (let i = 0; i < faceAlignments.length; i++) { + const alignedFace = faceAlignments[i]; + const faceDataOffset = i * faceSize * faceSize * 3; + warpAffineFloat32List( + image, + alignedFace, + faceSize, + faceData, + faceDataOffset, + ); + } + return faceData; +} diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index 081962935a..16c511c0b6 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -1,6 +1,10 @@ import mlIDbStorage from "services/face/db"; import { Face, MLSyncContext, Person } from "services/face/types"; -import FaceService from "../machineLearning/faceService"; +import { + getAllSyncedFacesMap, + runFaceClustering, + saveFaceCrop, +} from "./f-index"; import { fetchImageBitmap, getLocalFile } from "./image"; export const syncPeopleIndex = async (syncContext: MLSyncContext) => { @@ -12,10 +16,10 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => { // TODO: have faces addresable through fileId + faceId // to avoid index based addressing, which is prone to wrong results // one way could be to match nearest face within threshold in the file - const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext); + const allFacesMap = await getAllSyncedFacesMap(syncContext); const allFaces = [...allFacesMap.values()].flat(); - await FaceService.runFaceClustering(syncContext, allFaces); + await runFaceClustering(syncContext, allFaces); await syncPeopleFromClusters(syncContext, allFacesMap, allFaces); await mlIDbStorage.setIndexVersion("people", filesVersion); @@ -48,7 +52,7 @@ const syncPeopleFromClusters = async ( if (personFace && !personFace.crop?.cacheKey) { const file = await getLocalFile(personFace.fileId); const imageBitmap = await fetchImageBitmap(file); - await FaceService.saveFaceCrop(imageBitmap, personFace); + await saveFaceCrop(imageBitmap, personFace); } const person: Person = { diff --git a/web/apps/photos/src/services/machineLearning/faceService.ts b/web/apps/photos/src/services/machineLearning/faceService.ts deleted file mode 100644 index 74d7f7f246..0000000000 --- a/web/apps/photos/src/services/machineLearning/faceService.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { openCache } from "@/next/blob-cache"; -import log from "@/next/log"; -import { faceAlignment } from "services/face/align"; -import mlIDbStorage from "services/face/db"; -import { detectFaces, getRelativeDetection } from "services/face/detect"; -import { faceEmbeddings, mobileFaceNetFaceSize } from "services/face/embed"; -import { - DetectedFace, - Face, - MLSyncContext, - MLSyncFileContext, - type FaceAlignment, -} from "services/face/types"; -import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; -import { detectBlur } from "../face/blur"; -import { clusterFaces } from "../face/cluster"; -import { getFaceCrop } from "../face/crop"; -import { - fetchImageBitmap, - fetchImageBitmapForContext, - getFaceId, - getLocalFile, -} from "../face/image"; - -class FaceService { - async syncFileFaceDetections( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { - const { newMlFile } = fileContext; - newMlFile.faceDetectionMethod = { - value: "YoloFace", - version: 1, - }; - fileContext.newDetection = true; - const imageBitmap = await fetchImageBitmapForContext(fileContext); - const faceDetections = await detectFaces(imageBitmap); - // TODO: reenable faces filtering based on width - const detectedFaces = faceDetections?.map((detection) => { - return { - fileId: fileContext.enteFile.id, - detection, - } as DetectedFace; - }); - newMlFile.faces = detectedFaces?.map((detectedFace) => ({ - ...detectedFace, - id: getFaceId(detectedFace, newMlFile.imageDimensions), - })); - // ?.filter((f) => - // f.box.width > syncContext.config.faceDetection.minFaceSize - // ); - log.info("[MLService] Detected Faces: ", newMlFile.faces?.length); - } - - async syncFileFaceCrops( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { - const { newMlFile } = fileContext; - const imageBitmap = await fetchImageBitmapForContext(fileContext); - newMlFile.faceCropMethod = { - value: "ArcFace", - version: 1, - }; - - for (const face of newMlFile.faces) { - await this.saveFaceCrop(imageBitmap, face); - } - } - - async syncFileFaceAlignments( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ): Promise { - const { newMlFile } = fileContext; - newMlFile.faceAlignmentMethod = { - value: "ArcFace", - version: 1, - }; - fileContext.newAlignment = true; - const imageBitmap = - fileContext.imageBitmap || - (await fetchImageBitmapForContext(fileContext)); - - // Execute the face alignment calculations - for (const face of newMlFile.faces) { - face.alignment = faceAlignment(face.detection); - } - // Extract face images and convert to Float32Array - const faceAlignments = newMlFile.faces.map((f) => f.alignment); - const faceImages = await extractFaceImagesToFloat32( - faceAlignments, - mobileFaceNetFaceSize, - imageBitmap, - ); - const blurValues = detectBlur(faceImages, newMlFile.faces); - newMlFile.faces.forEach((f, i) => (f.blurValue = blurValues[i])); - - imageBitmap.close(); - log.info("[MLService] alignedFaces: ", newMlFile.faces?.length); - - return faceImages; - } - - async syncFileFaceEmbeddings( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - alignedFacesInput: Float32Array, - ) { - const { newMlFile } = fileContext; - newMlFile.faceEmbeddingMethod = { - value: "MobileFaceNet", - version: 2, - }; - // TODO: when not storing face crops, image will be needed to extract faces - // fileContext.imageBitmap || - // (await this.getImageBitmap(fileContext)); - - const embeddings = await faceEmbeddings(alignedFacesInput); - newMlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); - - log.info("[MLService] facesWithEmbeddings: ", newMlFile.faces.length); - } - - async syncFileFaceMakeRelativeDetections( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { - const { newMlFile } = fileContext; - for (let i = 0; i < newMlFile.faces.length; i++) { - const face = newMlFile.faces[i]; - if (face.detection.box.x + face.detection.box.width < 2) continue; // Skip if somehow already relative - face.detection = getRelativeDetection( - face.detection, - newMlFile.imageDimensions, - ); - } - } - - async saveFaceCrop(imageBitmap: ImageBitmap, face: Face) { - const faceCrop = getFaceCrop(imageBitmap, face.detection); - - const blob = await imageBitmapToBlob(faceCrop.image); - - const cache = await openCache("face-crops"); - await cache.put(face.id, blob); - - faceCrop.image.close(); - - return blob; - } - - async getAllSyncedFacesMap(syncContext: MLSyncContext) { - if (syncContext.allSyncedFacesMap) { - return syncContext.allSyncedFacesMap; - } - - syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap(); - return syncContext.allSyncedFacesMap; - } - - public async runFaceClustering( - syncContext: MLSyncContext, - allFaces: Array, - ) { - // await this.init(); - - if (!allFaces || allFaces.length < 50) { - log.info( - `Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`, - ); - return; - } - - log.info("Running clustering allFaces: ", allFaces.length); - syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( - allFaces.map((f) => Array.from(f.embedding)), - ); - syncContext.mlLibraryData.faceClusteringMethod = { - value: "Hdbscan", - version: 1, - }; - log.info( - "[MLService] Got face clustering results: ", - JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), - ); - - // syncContext.faceClustersWithNoise = { - // clusters: syncContext.faceClusteringResults.clusters.map( - // (faces) => ({ - // faces, - // }) - // ), - // noise: syncContext.faceClusteringResults.noise, - // }; - } - - public async regenerateFaceCrop(faceID: string) { - const fileID = Number(faceID.split("-")[0]); - const personFace = await mlIDbStorage.getFace(fileID, faceID); - if (!personFace) { - throw Error("Face not found"); - } - - const file = await getLocalFile(personFace.fileId); - const imageBitmap = await fetchImageBitmap(file); - return await this.saveFaceCrop(imageBitmap, personFace); - } -} - -export default new FaceService(); - -async function extractFaceImagesToFloat32( - faceAlignments: Array, - faceSize: number, - image: ImageBitmap, -): Promise { - const faceData = new Float32Array( - faceAlignments.length * faceSize * faceSize * 3, - ); - for (let i = 0; i < faceAlignments.length; i++) { - const alignedFace = faceAlignments[i]; - const faceDataOffset = i * faceSize * faceSize * 3; - warpAffineFloat32List( - image, - alignedFace, - faceSize, - faceData, - faceDataOffset, - ); - } - return faceData; -} diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 9ff1e1e249..5387c92e13 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -26,9 +26,9 @@ import { import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; +import { regenerateFaceCrop, syncFileAnalyzeFaces } from "../face/f-index"; import { fetchImageBitmapForContext } from "../face/image"; import { syncPeopleIndex } from "../face/people"; -import FaceService from "./faceService"; /** * TODO-ML(MR): What and why. @@ -222,7 +222,7 @@ class MachineLearningService { faceID: string, ) { await downloadManager.init(APPS.PHOTOS, { token }); - return FaceService.regenerateFaceCrop(faceID); + return regenerateFaceCrop(faceID); } private newMlData(fileId: number) { @@ -467,9 +467,7 @@ class MachineLearningService { try { await fetchImageBitmapForContext(fileContext); - await Promise.all([ - this.syncFileAnalyzeFaces(syncContext, fileContext), - ]); + await Promise.all([syncFileAnalyzeFaces(syncContext, fileContext)]); newMlFile.errorCount = 0; newMlFile.lastErrorMessage = undefined; await this.persistOnServer(newMlFile, enteFile); @@ -548,40 +546,6 @@ class MachineLearningService { await this.persistMLLibraryData(syncContext); } - - private async syncFileAnalyzeFaces( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, - ) { - const { newMlFile } = fileContext; - const startTime = Date.now(); - await FaceService.syncFileFaceDetections(syncContext, fileContext); - - if (newMlFile.faces && newMlFile.faces.length > 0) { - await FaceService.syncFileFaceCrops(syncContext, fileContext); - - const alignedFacesData = await FaceService.syncFileFaceAlignments( - syncContext, - fileContext, - ); - - await FaceService.syncFileFaceEmbeddings( - syncContext, - fileContext, - alignedFacesData, - ); - - await FaceService.syncFileFaceMakeRelativeDetections( - syncContext, - fileContext, - ); - } - log.info( - `face detection time taken ${fileContext.enteFile.id}`, - Date.now() - startTime, - "ms", - ); - } } export default new MachineLearningService(); From b1d9da663e6bc20bc39fd3f4a072dc86b83e9f33 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:19:48 +0530 Subject: [PATCH 37/47] Prune --- web/apps/photos/src/services/face/types.ts | 54 ------------------- .../machineLearning/machineLearningService.ts | 42 --------------- 2 files changed, 96 deletions(-) diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index 40aa29aee8..eb366f0e21 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -136,60 +136,6 @@ export interface MlFileData { lastErrorMessage?: string; } -export interface FaceDetectionConfig { - method: FaceDetectionMethod; -} - -export interface FaceCropConfig { - enabled: boolean; - method: FaceCropMethod; - padding: number; - maxSize: number; - blobOptions: { - type: string; - quality: number; - }; -} - -export interface FaceAlignmentConfig { - method: FaceAlignmentMethod; -} - -export interface BlurDetectionConfig { - method: BlurDetectionMethod; - threshold: number; -} - -export interface FaceEmbeddingConfig { - method: FaceEmbeddingMethod; - faceSize: number; - generateTsne?: boolean; -} - -export declare type TSNEMetric = "euclidean" | "manhattan"; - -export interface TSNEConfig { - samples: number; - dim: number; - perplexity?: number; - earlyExaggeration?: number; - learningRate?: number; - nIter?: number; - metric?: TSNEMetric; -} - -export interface MLSyncConfig { - batchSize: number; - imageSource: ImageType; - faceDetection: FaceDetectionConfig; - faceCrop: FaceCropConfig; - faceAlignment: FaceAlignmentConfig; - blurDetection: BlurDetectionConfig; - faceEmbedding: FaceEmbeddingConfig; - faceClustering: any; - mlVersion: number; -} - export interface MLSearchConfig { enabled: boolean; } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 5387c92e13..35923ee5e4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -17,7 +17,6 @@ import { Landmark, MLLibraryData, MLSearchConfig, - MLSyncConfig, MLSyncContext, MLSyncFileContext, MLSyncResult, @@ -38,47 +37,6 @@ export const defaultMLVersion = 3; const batchSize = 200; -export const DEFAULT_ML_SYNC_CONFIG: MLSyncConfig = { - batchSize: 200, - imageSource: "Original", - faceDetection: { - method: "YoloFace", - }, - faceCrop: { - enabled: true, - method: "ArcFace", - padding: 0.25, - maxSize: 256, - blobOptions: { - type: "image/jpeg", - quality: 0.8, - }, - }, - faceAlignment: { - method: "ArcFace", - }, - blurDetection: { - method: "Laplacian", - threshold: 15, - }, - faceEmbedding: { - method: "MobileFaceNet", - faceSize: 112, - generateTsne: true, - }, - faceClustering: { - method: "Hdbscan", - minClusterSize: 3, - minSamples: 5, - clusterSelectionEpsilon: 0.6, - clusterSelectionMethod: "leaf", - minInputSize: 50, - // maxDistanceInsideCluster: 0.4, - generateDebugInfo: true, - }, - mlVersion: defaultMLVersion, -}; - export const MAX_ML_SYNC_ERROR_COUNT = 1; export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { From a2931414446ffab73830b1e381bdbb90546a9e34 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:21:38 +0530 Subject: [PATCH 38/47] Remove unused contexty --- web/apps/photos/src/services/face/f-index.ts | 35 +++++-------------- .../machineLearning/machineLearningService.ts | 2 +- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 9f282cdced..cee2bbc5c3 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,30 +22,20 @@ import { getLocalFile, } from "./image"; -export const syncFileAnalyzeFaces = async ( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, -) => { +export const syncFileAnalyzeFaces = async (fileContext: MLSyncFileContext) => { const { newMlFile } = fileContext; const startTime = Date.now(); - await syncFileFaceDetections(syncContext, fileContext); + await syncFileFaceDetections(fileContext); if (newMlFile.faces && newMlFile.faces.length > 0) { - await syncFileFaceCrops(syncContext, fileContext); + await syncFileFaceCrops(fileContext); - const alignedFacesData = await syncFileFaceAlignments( - syncContext, - fileContext, - ); + const alignedFacesData = await syncFileFaceAlignments(fileContext); - await syncFileFaceEmbeddings( - syncContext, - fileContext, - alignedFacesData, - ); + await syncFileFaceEmbeddings(fileContext, alignedFacesData); - await syncFileFaceMakeRelativeDetections(syncContext, fileContext); + await syncFileFaceMakeRelativeDetections(fileContext); } log.debug( () => @@ -53,10 +43,7 @@ export const syncFileAnalyzeFaces = async ( ); }; -const syncFileFaceDetections = async ( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, -) => { +const syncFileFaceDetections = async (fileContext: MLSyncFileContext) => { const { newMlFile } = fileContext; newMlFile.faceDetectionMethod = { value: "YoloFace", @@ -82,10 +69,7 @@ const syncFileFaceDetections = async ( log.info("[MLService] Detected Faces: ", newMlFile.faces?.length); }; -const syncFileFaceCrops = async ( - syncContext: MLSyncContext, - fileContext: MLSyncFileContext, -) => { +const syncFileFaceCrops = async (fileContext: MLSyncFileContext) => { const { newMlFile } = fileContext; const imageBitmap = await fetchImageBitmapForContext(fileContext); newMlFile.faceCropMethod = { @@ -99,7 +83,6 @@ const syncFileFaceCrops = async ( }; const syncFileFaceAlignments = async ( - syncContext: MLSyncContext, fileContext: MLSyncFileContext, ): Promise => { const { newMlFile } = fileContext; @@ -133,7 +116,6 @@ const syncFileFaceAlignments = async ( }; const syncFileFaceEmbeddings = async ( - syncContext: MLSyncContext, fileContext: MLSyncFileContext, alignedFacesInput: Float32Array, ) => { @@ -153,7 +135,6 @@ const syncFileFaceEmbeddings = async ( }; const syncFileFaceMakeRelativeDetections = async ( - syncContext: MLSyncContext, fileContext: MLSyncFileContext, ) => { const { newMlFile } = fileContext; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 35923ee5e4..5ed66f9aed 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -425,7 +425,7 @@ class MachineLearningService { try { await fetchImageBitmapForContext(fileContext); - await Promise.all([syncFileAnalyzeFaces(syncContext, fileContext)]); + await Promise.all([syncFileAnalyzeFaces(fileContext)]); newMlFile.errorCount = 0; newMlFile.lastErrorMessage = undefined; await this.persistOnServer(newMlFile, enteFile); From fb53ae71630b22f30a83ef5bef960f60797099ca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:27:28 +0530 Subject: [PATCH 39/47] Unwrap --- .../src/services/machineLearning/machineLearningService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 5ed66f9aed..808ff8fe00 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -425,7 +425,7 @@ class MachineLearningService { try { await fetchImageBitmapForContext(fileContext); - await Promise.all([syncFileAnalyzeFaces(fileContext)]); + await syncFileAnalyzeFaces(fileContext); newMlFile.errorCount = 0; newMlFile.lastErrorMessage = undefined; await this.persistOnServer(newMlFile, enteFile); From 865ddc0fa9805fc3e6cfa43c753e14fe5c1dca6e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:29:09 +0530 Subject: [PATCH 40/47] Inline --- web/apps/photos/src/services/face/f-index.ts | 47 ------------------- web/apps/photos/src/services/face/people.ts | 48 +++++++++++++++++--- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index cee2bbc5c3..db054ac293 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -7,13 +7,11 @@ import { faceEmbeddings, mobileFaceNetFaceSize } from "services/face/embed"; import { DetectedFace, Face, - MLSyncContext, MLSyncFileContext, type FaceAlignment, } from "services/face/types"; import { imageBitmapToBlob, warpAffineFloat32List } from "utils/image"; import { detectBlur } from "./blur"; -import { clusterFaces } from "./cluster"; import { getFaceCrop } from "./crop"; import { fetchImageBitmap, @@ -161,51 +159,6 @@ export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { return blob; }; -export const getAllSyncedFacesMap = async (syncContext: MLSyncContext) => { - if (syncContext.allSyncedFacesMap) { - return syncContext.allSyncedFacesMap; - } - - syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap(); - return syncContext.allSyncedFacesMap; -}; - -export const runFaceClustering = async ( - syncContext: MLSyncContext, - allFaces: Array, -) => { - // await this.init(); - - if (!allFaces || allFaces.length < 50) { - log.info( - `Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`, - ); - return; - } - - log.info("Running clustering allFaces: ", allFaces.length); - syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( - allFaces.map((f) => Array.from(f.embedding)), - ); - syncContext.mlLibraryData.faceClusteringMethod = { - value: "Hdbscan", - version: 1, - }; - log.info( - "[MLService] Got face clustering results: ", - JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), - ); - - // syncContext.faceClustersWithNoise = { - // clusters: syncContext.faceClusteringResults.clusters.map( - // (faces) => ({ - // faces, - // }) - // ), - // noise: syncContext.faceClusteringResults.noise, - // }; -}; - export const regenerateFaceCrop = async (faceID: string) => { const fileID = Number(faceID.split("-")[0]); const personFace = await mlIDbStorage.getFace(fileID, faceID); diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index 16c511c0b6..f795e94f16 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -1,10 +1,8 @@ +import log from "@/next/log"; import mlIDbStorage from "services/face/db"; import { Face, MLSyncContext, Person } from "services/face/types"; -import { - getAllSyncedFacesMap, - runFaceClustering, - saveFaceCrop, -} from "./f-index"; +import { clusterFaces } from "./cluster"; +import { saveFaceCrop } from "./f-index"; import { fetchImageBitmap, getLocalFile } from "./image"; export const syncPeopleIndex = async (syncContext: MLSyncContext) => { @@ -16,7 +14,9 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => { // TODO: have faces addresable through fileId + faceId // to avoid index based addressing, which is prone to wrong results // one way could be to match nearest face within threshold in the file - const allFacesMap = await getAllSyncedFacesMap(syncContext); + const allFacesMap = + syncContext.allSyncedFacesMap ?? + (syncContext.allSyncedFacesMap = await mlIDbStorage.getAllFacesMap()); const allFaces = [...allFacesMap.values()].flat(); await runFaceClustering(syncContext, allFaces); @@ -25,6 +25,42 @@ export const syncPeopleIndex = async (syncContext: MLSyncContext) => { await mlIDbStorage.setIndexVersion("people", filesVersion); }; +const runFaceClustering = async ( + syncContext: MLSyncContext, + allFaces: Array, +) => { + // await this.init(); + + if (!allFaces || allFaces.length < 50) { + log.info( + `Skipping clustering since number of faces (${allFaces.length}) is less than the clustering threshold (50)`, + ); + return; + } + + log.info("Running clustering allFaces: ", allFaces.length); + syncContext.mlLibraryData.faceClusteringResults = await clusterFaces( + allFaces.map((f) => Array.from(f.embedding)), + ); + syncContext.mlLibraryData.faceClusteringMethod = { + value: "Hdbscan", + version: 1, + }; + log.info( + "[MLService] Got face clustering results: ", + JSON.stringify(syncContext.mlLibraryData.faceClusteringResults), + ); + + // syncContext.faceClustersWithNoise = { + // clusters: syncContext.faceClusteringResults.clusters.map( + // (faces) => ({ + // faces, + // }) + // ), + // noise: syncContext.faceClusteringResults.noise, + // }; +}; + const syncPeopleFromClusters = async ( syncContext: MLSyncContext, allFacesMap: Map>, From fca668b8e6879d12cc3d83699e0aeb73c9ac671d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:34:24 +0530 Subject: [PATCH 41/47] Inline --- web/apps/photos/src/services/face/face.worker.ts | 12 +----------- web/apps/photos/src/services/face/types.ts | 15 --------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index cc4d7017c9..9176334756 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -1,14 +1,8 @@ -import log from "@/next/log"; import { expose } from "comlink"; -import { MachineLearningWorker } from "services/face/types"; import mlService from "services/machineLearning/machineLearningService"; import { EnteFile } from "types/file"; -export class DedicatedMLWorker implements MachineLearningWorker { - constructor() { - log.info("DedicatedMLWorker constructor called"); - } - +export class DedicatedMLWorker { public async closeLocalSyncContext() { return mlService.closeLocalSyncContext(); } @@ -33,10 +27,6 @@ export class DedicatedMLWorker implements MachineLearningWorker { ) { return mlService.regenerateFaceCrop(token, userID, faceID); } - - public close() { - self.close(); - } } expose(DedicatedMLWorker, self); diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index eb366f0e21..db163d349a 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -181,18 +181,3 @@ export interface MLLibraryData { } export declare type MLIndex = "files" | "people"; - -export interface MachineLearningWorker { - closeLocalSyncContext(): Promise; - - syncLocalFile( - token: string, - userID: number, - enteFile: EnteFile, - localFile: globalThis.File, - ); - - sync(token: string, userID: number): Promise; - - close(): void; -} From 23087ee8dc542edb1f989ddc4c246cf29074a85a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:37:02 +0530 Subject: [PATCH 42/47] Shuffle --- web/apps/photos/src/services/face/people.ts | 3 +- web/apps/photos/src/services/face/types.ts | 22 ------------ .../machineLearning/machineLearningService.ts | 34 +++++++++++++------ 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index f795e94f16..416ba9e4ec 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -1,6 +1,7 @@ import log from "@/next/log"; import mlIDbStorage from "services/face/db"; -import { Face, MLSyncContext, Person } from "services/face/types"; +import { Face, Person } from "services/face/types"; +import { type MLSyncContext } from "services/machineLearning/machineLearningService"; import { clusterFaces } from "./cluster"; import { saveFaceCrop } from "./f-index"; import { fetchImageBitmap, getLocalFile } from "./image"; diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index db163d349a..99244bf61d 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -1,4 +1,3 @@ -import PQueue from "p-queue"; import type { ClusterFacesResult } from "services/face/cluster"; import { Dimensions } from "services/face/geom"; import { EnteFile } from "types/file"; @@ -140,27 +139,6 @@ export interface MLSearchConfig { enabled: boolean; } -export interface MLSyncContext { - token: string; - userID: number; - - localFilesMap: Map; - outOfSyncFiles: EnteFile[]; - nSyncedFiles: number; - nSyncedFaces: number; - allSyncedFacesMap?: Map>; - - error?: Error; - - // oldMLLibraryData: MLLibraryData; - mlLibraryData: MLLibraryData; - - syncQueue: PQueue; - - getEnteWorker(id: number): Promise; - dispose(): Promise; -} - export interface MLSyncFileContext { enteFile: EnteFile; localFile?: globalThis.File; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 808ff8fe00..e601aa414e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -17,7 +17,6 @@ import { Landmark, MLLibraryData, MLSearchConfig, - MLSyncContext, MLSyncFileContext, MLSyncResult, MlFileData, @@ -59,6 +58,27 @@ export async function updateMLSearchConfig(newConfig: MLSearchConfig) { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); } +export interface MLSyncContext { + token: string; + userID: number; + + localFilesMap: Map; + outOfSyncFiles: EnteFile[]; + nSyncedFiles: number; + nSyncedFaces: number; + allSyncedFacesMap?: Map>; + + error?: Error; + + // oldMLLibraryData: MLLibraryData; + mlLibraryData: MLLibraryData; + + syncQueue: PQueue; + + getEnteWorker(id: number): Promise; + dispose(): Promise; +} + export class LocalMLSyncContext implements MLSyncContext { public token: string; public userID: number; @@ -371,11 +391,7 @@ class MachineLearningService { console.log( `Indexing ${enteFile.title ?? ""} ${enteFile.id}`, ); - const mlFileData = await this.syncFile( - syncContext, - enteFile, - localFile, - ); + const mlFileData = await this.syncFile(enteFile, localFile); syncContext.nSyncedFaces += mlFileData.faces?.length || 0; syncContext.nSyncedFiles += 1; return mlFileData; @@ -408,11 +424,7 @@ class MachineLearningService { } } - private async syncFile( - syncContext: MLSyncContext, - enteFile: EnteFile, - localFile?: globalThis.File, - ) { + private async syncFile(enteFile: EnteFile, localFile?: globalThis.File) { log.debug(() => ({ a: "Syncing file", enteFile })); const fileContext: MLSyncFileContext = { enteFile, localFile }; const oldMlFile = await this.getMLFileData(enteFile.id); From 24524677965ddad3ed09ec409c6f8e257a79ccfd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:42:15 +0530 Subject: [PATCH 43/47] Bypass --- web/apps/photos/src/services/face/face.worker.ts | 8 ++------ .../services/machineLearning/machineLearningService.ts | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index 9176334756..19e88acbfa 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -20,12 +20,8 @@ export class DedicatedMLWorker { return mlService.sync(token, userID); } - public async regenerateFaceCrop( - token: string, - userID: number, - faceID: string, - ) { - return mlService.regenerateFaceCrop(token, userID, faceID); + public async regenerateFaceCrop(token: string, faceID: string) { + return mlService.regenerateFaceCrop(token, faceID); } } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index e601aa414e..be3c61ff3d 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -194,11 +194,7 @@ class MachineLearningService { return mlSyncResult; } - public async regenerateFaceCrop( - token: string, - userID: number, - faceID: string, - ) { + public async regenerateFaceCrop(token: string, faceID: string) { await downloadManager.init(APPS.PHOTOS, { token }); return regenerateFaceCrop(faceID); } From f7099c9bfeef564d8a4a3cbbda0f98c6aa5ecbf8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:46:42 +0530 Subject: [PATCH 44/47] Prune --- web/apps/photos/src/components/ml/PeopleList.tsx | 3 --- web/apps/photos/src/services/face/face.worker.ts | 2 +- .../src/services/machineLearning/machineLearningService.ts | 7 +------ 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 534df6499a..9e5620a5c8 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -167,10 +167,7 @@ const FaceCropImageView: React.FC = ({ .legacyFaceCrop(faceID) /* cachedOrNew("face-crops", cacheKey, async () => { - const user = await ensureLocalUser(); return machineLearningService.regenerateFaceCrop( - user.token, - user.id, faceId, ); })*/ diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index 19e88acbfa..a12b7b543a 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -21,7 +21,7 @@ export class DedicatedMLWorker { } public async regenerateFaceCrop(token: string, faceID: string) { - return mlService.regenerateFaceCrop(token, faceID); + return mlService.regenerateFaceCrop(faceID); } } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index be3c61ff3d..9f37398924 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,14 +1,12 @@ import { haveWindow } from "@/next/env"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { APPS } from "@ente/shared/apps/constants"; import ComlinkCryptoWorker, { getDedicatedCryptoWorker, } from "@ente/shared/crypto"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import downloadManager from "services/download"; import { putEmbedding } from "services/embeddingService"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db"; import { @@ -158,8 +156,6 @@ class MachineLearningService { throw Error("Token needed by ml service to sync file"); } - await downloadManager.init(APPS.PHOTOS, { token }); - const syncContext = await this.getSyncContext(token, userID); await this.syncLocalFiles(syncContext); @@ -194,8 +190,7 @@ class MachineLearningService { return mlSyncResult; } - public async regenerateFaceCrop(token: string, faceID: string) { - await downloadManager.init(APPS.PHOTOS, { token }); + public async regenerateFaceCrop(faceID: string) { return regenerateFaceCrop(faceID); } From 3603ca3d9bf04a707b77f6ef8893f94e7bbdbada Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 14:52:36 +0530 Subject: [PATCH 45/47] Reintroduce in worker --- web/apps/photos/src/services/face/face.worker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index a12b7b543a..6b5e17910e 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -1,6 +1,8 @@ import { expose } from "comlink"; import mlService from "services/machineLearning/machineLearningService"; import { EnteFile } from "types/file"; +import downloadManager from "services/download"; +import { APPS } from "@ente/shared/apps/constants"; export class DedicatedMLWorker { public async closeLocalSyncContext() { @@ -17,10 +19,12 @@ export class DedicatedMLWorker { } public async sync(token: string, userID: number) { + await downloadManager.init(APPS.PHOTOS, { token }); return mlService.sync(token, userID); } public async regenerateFaceCrop(token: string, faceID: string) { + await downloadManager.init(APPS.PHOTOS, { token }); return mlService.regenerateFaceCrop(faceID); } } From 7156a42d9248ee6cfd3da651a3c4ea9d57506cce Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 15:06:02 +0530 Subject: [PATCH 46/47] Unused --- web/apps/photos/src/services/machineLearning/mlWorkManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 8f58965f14..1cb61af00e 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -311,11 +311,11 @@ class MLWorkManager { } } - public stopSyncJob(terminateWorker: boolean = true) { + public stopSyncJob() { try { log.info("MLWorkManager.stopSyncJob"); this.mlSyncJob?.stop(); - terminateWorker && this.terminateSyncJobWorker(); + this.terminateSyncJobWorker(); } catch (e) { log.error("Failed to stop MLSync Job", e); } From 33272776d1db5c661606193b70c25441460dbb40 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 16 May 2024 15:07:15 +0530 Subject: [PATCH 47/47] lf --- web/apps/photos/src/services/face/face.worker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/face.worker.ts b/web/apps/photos/src/services/face/face.worker.ts index 6b5e17910e..8083406bf1 100644 --- a/web/apps/photos/src/services/face/face.worker.ts +++ b/web/apps/photos/src/services/face/face.worker.ts @@ -1,8 +1,8 @@ +import { APPS } from "@ente/shared/apps/constants"; import { expose } from "comlink"; +import downloadManager from "services/download"; import mlService from "services/machineLearning/machineLearningService"; import { EnteFile } from "types/file"; -import downloadManager from "services/download"; -import { APPS } from "@ente/shared/apps/constants"; export class DedicatedMLWorker { public async closeLocalSyncContext() {