diff --git a/web/apps/photos/src/components/ml/MLSearchSettings.tsx b/web/apps/photos/src/components/ml/MLSearchSettings.tsx index 7d67514066..a2523ba962 100644 --- a/web/apps/photos/src/components/ml/MLSearchSettings.tsx +++ b/web/apps/photos/src/components/ml/MLSearchSettings.tsx @@ -1,4 +1,4 @@ -import { canEnableFaceIndexing } from "@/new/photos/services/ml/indexer"; +import { canEnableFaceIndexing } from "@/new/photos/services/ml"; import log from "@/next/log"; import { EnteMenuItem } from "@ente/shared/components/Menu/EnteMenuItem"; import { diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 83c7067457..d06c3cde1d 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -1,4 +1,4 @@ -import { unidentifiedFaceIDs } from "@/new/photos/services/ml/indexer"; +import { unidentifiedFaceIDs } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/people"; import { EnteFile } from "@/new/photos/types/file"; import { blobCache } from "@/next/blob-cache"; diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 0109da7f8f..9622aad32e 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -2,7 +2,7 @@ import DownloadManager from "@/new/photos/services/download"; import { isFaceIndexingEnabled, setIsFaceIndexingEnabled, -} from "@/new/photos/services/ml/indexer"; +} from "@/new/photos/services/ml"; import mlWorkManager from "@/new/photos/services/ml/mlWorkManager"; import { clientPackageName, staticAppTitle } from "@/next/app"; import { CustomHead } from "@/next/components/Head"; diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 2013553fe7..78720b4380 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -1,7 +1,7 @@ import DownloadManager from "@/new/photos/services/download"; import { clearFeatureFlagSessionState } from "@/new/photos/services/feature-flags"; -import { terminateFaceWorker } from "@/new/photos/services/ml"; -import { clearFaceData } from "@/new/photos/services/ml/db"; +import { terminateMLWorker } from "@/new/photos/services/ml"; +import { clearFaceDB } from "@/new/photos/services/ml/db"; import mlWorkManager from "@/new/photos/services/ml/mlWorkManager"; import log from "@/next/log"; import { accountLogout } from "@ente/accounts/services/logout"; @@ -19,11 +19,23 @@ export const photosLogout = async () => { const ignoreError = (label: string, e: unknown) => log.error(`Ignoring error during logout (${label})`, e); + // - Workers + // Terminate any workers before clearing persistent state. // See: [Note: Caching IDB instances in separate execution contexts]. + try { + terminateMLWorker(); + } catch (e) { + ignoreError("face", e); + } + + // - Remote logout and clear state + await accountLogout(); + // - Photos specific logout + try { clearFeatureFlagSessionState(); } catch (e) { @@ -42,11 +54,7 @@ export const photosLogout = async () => { ignoreError("CLIP", e); } - try { - terminateFaceWorker(); - } catch (e) { - ignoreError("face", e); - } + // - Desktop const electron = globalThis.electron; if (electron) { @@ -57,7 +65,7 @@ export const photosLogout = async () => { } try { - await clearFaceData(); + await clearFaceDB(); } catch (e) { ignoreError("face", e); } @@ -69,7 +77,7 @@ export const photosLogout = async () => { } try { - await electron?.logout(); + await electron.logout(); } catch (e) { ignoreError("electron", e); } diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 2b0482f07f..17eb1663ce 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,7 +2,7 @@ import { FILE_TYPE } from "@/media/file-type"; import { faceIndexingStatus, isFaceIndexingEnabled, -} from "@/new/photos/services/ml/indexer"; +} from "@/new/photos/services/ml"; import mlWorkManager from "@/new/photos/services/ml/mlWorkManager"; import type { Person } from "@/new/photos/services/ml/people"; import { EnteFile } from "@/new/photos/types/file"; diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 693e087cd6..575cb0a9c7 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,5 +1,5 @@ import { FILE_TYPE } from "@/media/file-type"; -import type { FaceIndexingStatus } from "@/new/photos/services/ml/indexer"; +import type { FaceIndexingStatus } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/people"; import { EnteFile } from "@/new/photos/types/file"; import { City } from "services/locationSearchService"; diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.ts index 1e305bbe7c..cd0cafed85 100644 --- a/web/packages/new/photos/services/ml/crop.ts +++ b/web/packages/new/photos/services/ml/crop.ts @@ -1,7 +1,7 @@ import type { Box } from "@/new/photos/services/ml/types"; import { blobCache } from "@/next/blob-cache"; import { ensure } from "@/utils/ensure"; -import type { FaceAlignment } from "./f-index"; +import type { FaceAlignment } from "./index-face"; export const saveFaceCrop = async ( imageBitmap: ImageBitmap, diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index 4e3eb74dab..8eeda67992 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -3,7 +3,7 @@ import { deleteDB, openDB, type DBSchema } from "idb"; import type { FaceIndex } from "./types"; /** - * [Note: Face DB schema] + * Face DB schema. * * There "face" database is made of two object stores: * @@ -62,20 +62,9 @@ interface FileStatus { } /** - * A promise to the face DB. + * A lazily-created, cached promise for face DB. * - * We open the database once (lazily), and thereafter save and reuse the promise - * each time something wants to connect to it. - * - * This promise can subsequently get cleared if we need to relinquish our - * connection (e.g. if another client wants to open the face DB with a newer - * version of the schema). - * - * Note that this is module specific state, so the main thread and each worker - * thread that calls the functions in this module will have their own promises. - * To ensure that all connections get torn down correctly, we need to call - * {@link closeFaceDBConnectionsIfNeeded} from both the main thread and all the - * worker threads that use this module. + * See: [Note: Caching IDB instances in separate execution contexts]. */ let _faceDB: ReturnType | undefined; @@ -125,28 +114,20 @@ const deleteLegacyDB = () => { const faceDB = () => (_faceDB ??= openFaceDB()); /** - * Close the face DB connection (if any) opened by this module. + * Clear any data stored in the face DB. * - * To ensure proper teardown of the DB connections, this function must be called - * at least once by any execution context that has called any of the other - * functions in this module. + * This is meant to be called during logout in the main thread. */ -export const closeFaceDBConnectionsIfNeeded = async () => { +export const clearFaceDB = async () => { + deleteLegacyDB(); + try { if (_faceDB) (await _faceDB).close(); - } finally { - _faceDB = undefined; + } catch (e) { + log.warn("Ignoring error when trying to close face DB", e); } -}; + _faceDB = undefined; -/** - * Clear any data stored by the face module. - * - * Meant to be called during logout. - */ -export const clearFaceData = async () => { - deleteLegacyDB(); - await closeFaceDBConnectionsIfNeeded(); return deleteDB("face", { blocked() { log.warn( diff --git a/web/packages/new/photos/services/ml/embedding.ts b/web/packages/new/photos/services/ml/embedding.ts index bfb104c0f5..42047dd326 100644 --- a/web/packages/new/photos/services/ml/embedding.ts +++ b/web/packages/new/photos/services/ml/embedding.ts @@ -13,7 +13,7 @@ import log from "@/next/log"; import { apiURL } from "@/next/origins"; import { z } from "zod"; import { saveFaceIndex } from "./db"; -import { faceIndexingVersion } from "./f-index"; +import { faceIndexingVersion } from "./index-face"; import { type FaceIndex } from "./types"; /** diff --git a/web/packages/new/photos/services/ml/image.ts b/web/packages/new/photos/services/ml/image.ts index d7875fa348..114247a91b 100644 --- a/web/packages/new/photos/services/ml/image.ts +++ b/web/packages/new/photos/services/ml/image.ts @@ -1,6 +1,4 @@ -// TODO: These arise from the array indexing in the pre-processing code. Isolate -// once that code settles down to its final place (currently duplicated across -// web and desktop). +// See: [Note: Allowing non-null assertions selectively] /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { ensure } from "@/utils/ensure"; diff --git a/web/packages/new/photos/services/ml/f-index.ts b/web/packages/new/photos/services/ml/index-face.ts similarity index 98% rename from web/packages/new/photos/services/ml/f-index.ts rename to web/packages/new/photos/services/ml/index-face.ts index c5ab400b9b..aeea87e041 100644 --- a/web/packages/new/photos/services/ml/f-index.ts +++ b/web/packages/new/photos/services/ml/index-face.ts @@ -1,7 +1,9 @@ -// The ML code in this file involves imperative array indexing and processing, -// and allowing non-null assertions ("!") are the easiest way to get tsc to -// accept it in the presence of noUncheckedIndexedAccess without obfuscating the -// original algorithms. +// [Note: Allowing non-null assertions selectively] +// +// The code in this file involves a lot of imperative array processing and +// indexing, and allowing non-null assertions ("!") is the easiest way to get +// TypeScript to accept it in the presence of noUncheckedIndexedAccess without +// obfuscating the original algorithms. // /* eslint-disable @typescript-eslint/no-non-null-assertion */ diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 743c1dbd61..3a8ddebea7 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -1,33 +1,165 @@ /** - * @file Main thread interface to {@link FaceWorker}. + * @file Main thread interface to {@link MLWorker}. */ +import { + isBetaUser, + isInternalUser, +} from "@/new/photos/services/feature-flags"; +import { + getAllLocalFiles, + getLocalTrashedFiles, +} from "@/new/photos/services/files"; +import { + faceIndex, + indexableFileIDs, + indexedAndIndexableCounts, + updateAssumingLocalFiles, +} from "@/new/photos/services/ml/db"; +import type { EnteFile } from "@/new/photos/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { FaceWorker } from "./worker"; +import { ensure } from "@/utils/ensure"; +import { MLWorker } from "./worker"; /** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ -let _comlinkWorker: ComlinkWorker | undefined; +let _comlinkWorker: ComlinkWorker | undefined; -/** Lazily created, cached, instance of {@link FaceWorker}. */ -export const faceWorker = async () => +/** Lazily created, cached, instance of {@link MLWorker}. */ +export const worker = async () => (_comlinkWorker ??= createComlinkWorker()).remote; const createComlinkWorker = () => - new ComlinkWorker( - "face", + new ComlinkWorker( + "ml", new Worker(new URL("worker.ts", import.meta.url)), ); /** - * Terminate {@link faceWorker} (if any). + * Terminate {@link worker} (if any). * - * This is useful during logout to immediately stop any background face related - * operations that are in-flight for the current user. After the user logs in - * again, a new {@link faceWorker} will be created on demand. + * This is useful during logout to immediately stop any background ML operations + * that are in-flight for the current user. After the user logs in again, a new + * {@link worker} will be created on demand for subsequent usage. */ -export const terminateFaceWorker = () => { +export const terminateMLWorker = () => { if (_comlinkWorker) { _comlinkWorker.terminate(); _comlinkWorker = undefined; } }; + +export interface FaceIndexingStatus { + /** + * Which phase we are in within the indexing pipeline when viewed across the + * user's entire library: + * + * - "scheduled": There are files we know of that have not been indexed. + * + * - "indexing": The face indexer is currently running. + * + * - "clustering": All files we know of have been indexed, and we are now + * clustering the faces that were found. + * + * - "done": Face indexing and clustering is complete for the user's + * library. + */ + phase: "scheduled" | "indexing" | "clustering" | "done"; + /** The number of files that have already been indexed. */ + nSyncedFiles: number; + /** The total number of files that are eligible for indexing. */ + nTotalFiles: number; +} + +export const faceIndexingStatus = async ( + isSyncing: boolean, +): Promise => { + const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); + + let phase: FaceIndexingStatus["phase"]; + if (indexableCount > 0) { + if (!isSyncing) { + phase = "scheduled"; + } else { + phase = "indexing"; + } + } else { + phase = "done"; + } + + return { + phase, + nSyncedFiles: indexedCount, + nTotalFiles: indexableCount + indexedCount, + }; +}; + +/** + * Return the IDs of all the faces in the given {@link enteFile} that are not + * associated with a person cluster. + */ +export const unidentifiedFaceIDs = async ( + enteFile: EnteFile, +): Promise => { + const index = await faceIndex(enteFile.id); + return index?.faceEmbedding.faces.map((f) => f.faceID) ?? []; +}; + +/** + * Return true if we should show an option to the user to allow them to enable + * face search in the UI. + */ +export const canEnableFaceIndexing = async () => + (await isInternalUser()) || (await isBetaUser()); + +/** + * Return true if the user has enabled face indexing in the app's settings. + * + * This setting is persisted locally (in local storage) and is not synced with + * remote. There is a separate setting, "faceSearchEnabled" that is synced with + * remote, but that tracks whether or not the user has enabled face search once + * on any client. This {@link isFaceIndexingEnabled} property, on the other + * hand, denotes whether or not indexing is enabled on the current client. + */ +export const isFaceIndexingEnabled = () => + localStorage.getItem("faceIndexingEnabled") == "1"; + +/** + * Update the (locally stored) value of {@link isFaceIndexingEnabled}. + */ +export const setIsFaceIndexingEnabled = (enabled: boolean) => + enabled + ? localStorage.setItem("faceIndexingEnabled", "1") + : localStorage.removeItem("faceIndexingEnabled"); + +/** + * Sync face DB with the local (and potentially indexable) files that we know + * about. Then return the next {@link count} files that still need to be + * indexed. + * + * For specifics of what a "sync" entails, see {@link updateAssumingLocalFiles}. + * + * @param userID Sync only files owned by a {@link userID} with the face DB. + * + * @param count Limit the resulting list of indexable files to {@link count}. + */ +export const syncWithLocalFilesAndGetFilesToIndex = async ( + userID: number, + count: number, +): Promise => { + const isIndexable = (f: EnteFile) => f.ownerID == userID; + + const localFiles = await getAllLocalFiles(); + const localFilesByID = new Map( + localFiles.filter(isIndexable).map((f) => [f.id, f]), + ); + + const localTrashFileIDs = (await getLocalTrashedFiles()).map((f) => f.id); + + await updateAssumingLocalFiles( + Array.from(localFilesByID.keys()), + localTrashFileIDs, + ); + + const fileIDsToIndex = await indexableFileIDs(count); + return fileIDsToIndex.map((id) => ensure(localFilesByID.get(id))); +}; diff --git a/web/packages/new/photos/services/ml/indexer.ts b/web/packages/new/photos/services/ml/indexer.ts deleted file mode 100644 index 51535e6d9d..0000000000 --- a/web/packages/new/photos/services/ml/indexer.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { - isBetaUser, - isInternalUser, -} from "@/new/photos/services/feature-flags"; -import { - getAllLocalFiles, - getLocalTrashedFiles, -} from "@/new/photos/services/files"; -import { - faceIndex, - indexableFileIDs, - indexedAndIndexableCounts, - updateAssumingLocalFiles, -} from "@/new/photos/services/ml/db"; -import type { EnteFile } from "@/new/photos/types/file"; -// import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { ensure } from "@/utils/ensure"; -// import type { Remote } from "comlink"; -// import type { FaceIndexerWorker } from "./indexer.worker"; - -/** - * Face indexing orchestrator. - * - * This module exposes a singleton instance of this class which drives the face - * indexing process on the user's library. - * - * The indexer operates in two modes - live indexing and backfill. - * - * When live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which puts them in a queue and indexes them one - * by one. This is more efficient since we already have the file's content at - * hand and do not have to download and decrypt it. - * - * When backfilling, the indexer figures out if any of the user's files - * (irrespective of where they were uploaded from) still need to be indexed, and - * if so, downloads, decrypts and indexes them. - * - * Live indexing has higher priority, backfilling runs otherwise. If nothing - * remains to be indexed, the indexer goes to sleep for a while. - */ -// eslint-disable-next-line @typescript-eslint/no-extraneous-class -class FaceIndexer { - /** Live indexing queue. */ - // private liveItems: { enteFile: EnteFile; file: File | undefined }[]; - /** Timeout for when the next time we will wake up. */ - // private wakeTimeout: ReturnType | undefined; - // /** - // * Add a file to the live indexing queue. - // * - // * @param enteFile An {@link EnteFile} that should be indexed. - // * - // * @param file The contents of {@link enteFile} as a web {@link File} - // * object, if available. - // */ - // enqueueFile(enteFile: EnteFile, file: File | undefined) { - // // If face indexing is not enabled, don't enqueue anything. Later on if - // // the user turns on face indexing these files will get indexed as part - // // of the backfilling anyway, the live indexing is just an optimization. - // if (!mlWorkManager.isMlSearchEnabled) return; - // this.liveItems.push({ enteFile, file }); - // this.wakeUpIfNeeded(); - // } - // private wakeUpIfNeeded() { - // // Already awake. - // if (!this.wakeTimeout) return; - // // Cancel the alarm, wake up now. - // clearTimeout(this.wakeTimeout); - // this.wakeTimeout = undefined; - // // Get to work. - // this.tick(); - // } - /* TODO-ML(MR): This code is not currently in use */ - /** - * A promise for the lazily created singleton {@link FaceIndexerWorker} remote - * exposed by this module. - */ - // _faceIndexer: Promise>; - /** - * Main thread interface to the face indexer. - * - * This function provides a promise that resolves to a lazily created singleton - * remote with a {@link FaceIndexerWorker} at the other end. - */ - // faceIndexer = (): Promise> => - // (this._faceIndexer ??= createFaceIndexerComlinkWorker().remote); - // private async tick() { - // console.log("tick"); - // const item = this.liveItems.pop(); - // if (!item) { - // // TODO-ML: backfill instead if needed here. - // this.wakeTimeout = setTimeout(() => { - // this.wakeTimeout = undefined; - // this.wakeUpIfNeeded(); - // }, 30 * 1000); - // return; - // } - // /* - // const fileID = item.enteFile.id; - // try { - // const faceIndex = await indexFaces(item.enteFile, item.file, userAgent); - // log.info(`faces in file ${fileID}`, faceIndex); - // } catch (e) { - // log.error(`Failed to index faces in file ${fileID}`, e); - // markIndexingFailed(item.enteFile.id); - // } - // */ - // // Let the runloop drain. - // await wait(0); - // // Run again. - // // TODO - // // this.tick(); - // } - /** - * Add a newly uploaded file to the face indexing queue. - * - * @param enteFile The {@link EnteFile} that was uploaded. - * @param file - */ - /* - indexFacesInFile = (enteFile: EnteFile, file: File) => { - if (!mlWorkManager.isMlSearchEnabled) return; - - faceIndexer().then((indexer) => { - indexer.enqueueFile(file, enteFile); - }); - }; - */ -} - -/** The singleton instance of {@link FaceIndexer}. */ -export default new FaceIndexer(); - -// const createFaceIndexerComlinkWorker = () => -// new ComlinkWorker( -// "face-indexer", -// new Worker(new URL("indexer.worker.ts", import.meta.url)), -// ); - -export interface FaceIndexingStatus { - /** - * Which phase we are in within the indexing pipeline when viewed across the - * user's entire library: - * - * - "scheduled": There are files we know of that have not been indexed. - * - * - "indexing": The face indexer is currently running. - * - * - "clustering": All files we know of have been indexed, and we are now - * clustering the faces that were found. - * - * - "done": Face indexing and clustering is complete for the user's - * library. - */ - phase: "scheduled" | "indexing" | "clustering" | "done"; - /** The number of files that have already been indexed. */ - nSyncedFiles: number; - /** The total number of files that are eligible for indexing. */ - nTotalFiles: number; -} - -export const faceIndexingStatus = async ( - isSyncing: boolean, -): Promise => { - const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); - - let phase: FaceIndexingStatus["phase"]; - if (indexableCount > 0) { - if (!isSyncing) { - phase = "scheduled"; - } else { - phase = "indexing"; - } - } else { - phase = "done"; - } - - return { - phase, - nSyncedFiles: indexedCount, - nTotalFiles: indexableCount + indexedCount, - }; -}; - -/** - * Return the IDs of all the faces in the given {@link enteFile} that are not - * associated with a person cluster. - */ -export const unidentifiedFaceIDs = async ( - enteFile: EnteFile, -): Promise => { - const index = await faceIndex(enteFile.id); - return index?.faceEmbedding.faces.map((f) => f.faceID) ?? []; -}; - -/** - * Return true if we should show an option to the user to allow them to enable - * face search in the UI. - */ -export const canEnableFaceIndexing = async () => - (await isInternalUser()) || (await isBetaUser()); - -/** - * Return true if the user has enabled face indexing in the app's settings. - * - * This setting is persisted locally (in local storage) and is not synced with - * remote. There is a separate setting, "faceSearchEnabled" that is synced with - * remote, but that tracks whether or not the user has enabled face search once - * on any client. This {@link isFaceIndexingEnabled} property, on the other - * hand, denotes whether or not indexing is enabled on the current client. - */ -export const isFaceIndexingEnabled = () => - localStorage.getItem("faceIndexingEnabled") == "1"; - -/** - * Update the (locally stored) value of {@link isFaceIndexingEnabled}. - */ -export const setIsFaceIndexingEnabled = (enabled: boolean) => - enabled - ? localStorage.setItem("faceIndexingEnabled", "1") - : localStorage.removeItem("faceIndexingEnabled"); - -/** - * Sync face DB with the local (and potentially indexable) files that we know - * about. Then return the next {@link count} files that still need to be - * indexed. - * - * For specifics of what a "sync" entails, see {@link updateAssumingLocalFiles}. - * - * @param userID Sync only files owned by a {@link userID} with the face DB. - * - * @param count Limit the resulting list of indexable files to {@link count}. - */ -export const syncWithLocalFilesAndGetFilesToIndex = async ( - userID: number, - count: number, -): Promise => { - const isIndexable = (f: EnteFile) => f.ownerID == userID; - - const localFiles = await getAllLocalFiles(); - const localFilesByID = new Map( - localFiles.filter(isIndexable).map((f) => [f.id, f]), - ); - - const localTrashFileIDs = (await getLocalTrashedFiles()).map((f) => f.id); - - await updateAssumingLocalFiles( - Array.from(localFilesByID.keys()), - localTrashFileIDs, - ); - - const fileIDsToIndex = await indexableFileIDs(count); - return fileIDsToIndex.map((id) => ensure(localFilesByID.get(id))); -}; diff --git a/web/packages/new/photos/services/ml/indexer.worker.ts b/web/packages/new/photos/services/ml/indexer.worker.ts deleted file mode 100644 index b5fd600a50..0000000000 --- a/web/packages/new/photos/services/ml/indexer.worker.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { - closeFaceDBConnectionsIfNeeded, - markIndexingFailed, - saveFaceIndex, -} from "@/new/photos/services/ml/db"; -import type { FaceIndex } from "@/new/photos/services/ml/types"; -import type { EnteFile } from "@/new/photos/types/file"; -import log from "@/next/log"; -import { fileLogID } from "../../utils/file"; -import { putFaceIndex } from "./embedding"; -import { indexFaces } from "./f-index"; - -/** - * Index faces in a file, save the persist the results locally, and put them on - * remote. - * - * This class is instantiated in a web worker so as to not get in the way of the - * main thread. It could've been a bunch of free standing functions too, it is - * just a class for convenience of compatibility with how the rest of our - * comlink workers are structured. - */ -export class FaceIndexerWorker { - /** - * Index faces in a file, save the persist the results locally, and put them - * on remote. - * - * @param enteFile The {@link EnteFile} to index. - * - * @param file If the file is one which is being uploaded from the current - * client, then we will also have access to the file's content. In such - * cases, pass a web {@link File} object to use that its data directly for - * face indexing. If this is not provided, then the file's contents will be - * downloaded and decrypted from remote. - * - * @param userAgent The UA of the client that is doing the indexing (us). - */ - async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { - const f = fileLogID(enteFile); - const startTime = Date.now(); - - let faceIndex: FaceIndex; - try { - faceIndex = await indexFaces(enteFile, file, userAgent); - } catch (e) { - // Mark indexing as having failed only if the indexing itself - // failed, not if there were subsequent failures (like when trying - // to put the result to remote or save it to the local face DB). - log.error(`Failed to index faces in ${f}`, e); - await markIndexingFailed(enteFile.id); - throw e; - } - - try { - await putFaceIndex(enteFile, faceIndex); - await saveFaceIndex(faceIndex); - } catch (e) { - log.error(`Failed to put/save face index for ${f}`, e); - throw e; - } - - log.debug(() => { - const nf = faceIndex.faceEmbedding.faces.length; - const ms = Date.now() - startTime; - return `Indexed ${nf} faces in ${f} (${ms} ms)`; - }); - - return faceIndex; - } - - /** - * Calls {@link closeFaceDBConnectionsIfNeeded} to close any open - * connections to the face DB from the web worker's context. - */ - closeFaceDB() { - void closeFaceDBConnectionsIfNeeded(); - } -} diff --git a/web/packages/new/photos/services/ml/machineLearningService.ts b/web/packages/new/photos/services/ml/machineLearningService.ts index cadd465b50..54c54dcce7 100644 --- a/web/packages/new/photos/services/ml/machineLearningService.ts +++ b/web/packages/new/photos/services/ml/machineLearningService.ts @@ -2,8 +2,8 @@ import type { EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import { syncWithLocalFilesAndGetFilesToIndex } from "./indexer"; -import { FaceIndexerWorker } from "./indexer.worker"; +import { syncWithLocalFilesAndGetFilesToIndex } from "."; +import { index } from "./worker"; const batchSize = 200; @@ -214,9 +214,7 @@ class MachineLearningService { file: File | undefined, userAgent: string, ) { - const worker = new FaceIndexerWorker(); - - await worker.index(enteFile, file, userAgent); + await index(enteFile, file, userAgent); } } diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 40c4cafec7..3fe960773a 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -1,14 +1,20 @@ -import { expose } from "comlink"; -import { pullFaceEmbeddings } from "./embedding"; +import { markIndexingFailed, saveFaceIndex } from "@/new/photos/services/ml/db"; +import type { FaceIndex } from "@/new/photos/services/ml/types"; +import type { EnteFile } from "@/new/photos/types/file"; +import log from "@/next/log"; +// import { expose } from "comlink"; +import { fileLogID } from "../../utils/file"; +import { pullFaceEmbeddings, putFaceIndex } from "./embedding"; +import { indexFaces } from "./index-face"; /** - * Run operations related to face indexing and search in a Web Worker. + * Run operations related to machine learning (e.g. indexing) in a Web Worker. * * This is a normal class that is however exposed (via comlink) as a proxy * running inside a Web Worker. This way, we do not bother the main thread with * tasks that might degrade interactivity. */ -export class FaceWorker { +export class MLWorker { private isSyncing = false; /** @@ -22,4 +28,56 @@ export class FaceWorker { } } -expose(FaceWorker); +// TODO-ML: Temorarily disable +// expose(MLWorker); + +/** + * Index faces in a file, save the persist the results locally, and put them + * on remote. + * + * @param enteFile The {@link EnteFile} to index. + * + * @param file If the file is one which is being uploaded from the current + * client, then we will also have access to the file's content. In such + * cases, pass a web {@link File} object to use that its data directly for + * face indexing. If this is not provided, then the file's contents will be + * downloaded and decrypted from remote. + * + * @param userAgent The UA of the client that is doing the indexing (us). + */ +export const index = async ( + enteFile: EnteFile, + file: File | undefined, + userAgent: string, +) => { + const f = fileLogID(enteFile); + const startTime = Date.now(); + + let faceIndex: FaceIndex; + try { + faceIndex = await indexFaces(enteFile, file, userAgent); + } catch (e) { + // Mark indexing as having failed only if the indexing itself + // failed, not if there were subsequent failures (like when trying + // to put the result to remote or save it to the local face DB). + log.error(`Failed to index faces in ${f}`, e); + await markIndexingFailed(enteFile.id); + throw e; + } + + try { + await putFaceIndex(enteFile, faceIndex); + await saveFaceIndex(faceIndex); + } catch (e) { + log.error(`Failed to put/save face index for ${f}`, e); + throw e; + } + + log.debug(() => { + const nf = faceIndex.faceEmbedding.faces.length; + const ms = Date.now() - startTime; + return `Indexed ${nf} faces in ${f} (${ms} ms)`; + }); + + return faceIndex; +};