From 8a031360c52f0775873916d281b5a0886e60ec64 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 12:09:30 +0530 Subject: [PATCH 01/16] Remove the debug scaffolding --- web/apps/photos/src/pages/cluster-debug.tsx | 581 ------------------ .../new/photos/components/MLSettings.tsx | 27 +- .../new/photos/services/ml/cluster.ts | 25 +- web/packages/new/photos/services/ml/index.ts | 5 +- web/packages/new/photos/services/ml/worker.ts | 9 +- 5 files changed, 39 insertions(+), 608 deletions(-) delete mode 100644 web/apps/photos/src/pages/cluster-debug.tsx diff --git a/web/apps/photos/src/pages/cluster-debug.tsx b/web/apps/photos/src/pages/cluster-debug.tsx deleted file mode 100644 index 1f6a1036f6..0000000000 --- a/web/apps/photos/src/pages/cluster-debug.tsx +++ /dev/null @@ -1,581 +0,0 @@ -import { SelectionBar } from "@/base/components/Navbar"; -import { pt } from "@/base/i18n"; -import { - faceCrop, - wipClusterDebugPageContents, - type ClusterDebugPageContents, -} from "@/new/photos/services/ml"; -import { - type ClusterFace, - type ClusteringOpts, - type ClusteringProgress, - type OnClusteringProgress, -} from "@/new/photos/services/ml/cluster"; -import { faceDirection } from "@/new/photos/services/ml/face"; -import type { EnteFile } from "@/new/photos/types/file"; -import { - FlexWrapper, - FluidContainer, - VerticallyCentered, -} from "@ente/shared/components/Container"; -import BackButton from "@mui/icons-material/ArrowBackOutlined"; -import { - Box, - Button, - Checkbox, - FormControlLabel, - IconButton, - LinearProgress, - Stack, - styled, - TextField, - Typography, -} from "@mui/material"; -import { useFormik, type FormikProps } from "formik"; -import { useRouter } from "next/router"; -import { AppContext } from "pages/_app"; -import React, { - memo, - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import AutoSizer from "react-virtualized-auto-sizer"; -import { - areEqual, - VariableSizeList, - type ListChildComponentProps, -} from "react-window"; - -// TODO-Cluster Temporary component for debugging -export default function ClusterDebug() { - const { startLoading, finishLoading, showNavBar } = useContext(AppContext); - - // The clustering result. - const [clusterRes, setClusterRes] = useState< - ClusterDebugPageContents | undefined - >(); - - // Keep the loading state callback as a ref instead of state to prevent - // rerendering when the progress gets updated during clustering. - const onProgressRef = useRef(); - - // Keep the form state at the top level otherwise it gets reset as we - // scroll. - const formik = useFormik({ - initialValues: { - minBlur: 10, - minScore: 0.8, - minClusterSize: 2, - joinThreshold: 0.76, - earlyExitThreshold: 0.9, - batchSize: 10000, - offsetIncrement: 7500, - badFaceHeuristics: true, - }, - onSubmit: (values) => - cluster( - { - minBlur: toFloat(values.minBlur), - minScore: toFloat(values.minScore), - minClusterSize: toFloat(values.minClusterSize), - joinThreshold: toFloat(values.joinThreshold), - earlyExitThreshold: toFloat(values.earlyExitThreshold), - batchSize: toFloat(values.batchSize), - offsetIncrement: toFloat(values.offsetIncrement), - badFaceHeuristics: values.badFaceHeuristics, - }, - (progress: ClusteringProgress) => - onProgressRef.current?.(progress), - ), - }); - - const cluster = useCallback( - async (opts: ClusteringOpts, onProgress: OnClusteringProgress) => { - setClusterRes(undefined); - startLoading(); - setClusterRes(await wipClusterDebugPageContents(opts, onProgress)); - finishLoading(); - }, - [startLoading, finishLoading], - ); - - useEffect(() => showNavBar(true), []); - - return ( - <> - - - {({ height, width }) => ( - - - - )} - - - - - ); -} - -// Formik converts nums to a string on edit. -const toFloat = (n: number | string) => - typeof n == "string" ? parseFloat(n) : n; - -const Options: React.FC = () => { - const router = useRouter(); - - const close = () => router.push("/gallery"); - - return ( - - - - - - {pt("Face Clusters")} - - - ); -}; - -const Container = styled("div")` - display: block; - flex: 1; - width: 100%; - flex-wrap: wrap; - overflow: hidden; - .pswp-thumbnail { - display: inline-block; - } -`; - -type OptionsFormProps = LoaderProps & { - formik: FormikProps; -}; - -const OptionsForm: React.FC = ({ formik, onProgressRef }) => { - return ( - - Parameters - - {formik.isSubmitting && } - - ); -}; - -const MemoizedForm = memo( - ({ - values, - handleSubmit, - handleChange, - isSubmitting, - }: FormikProps) => ( -
- - - - - - - - - - - - - } - label={ - - Bad face heuristics - - } - /> - - - -
- ), -); - -interface LoaderProps { - onProgressRef: React.MutableRefObject; -} - -const Loader: React.FC = ({ onProgressRef }) => { - const [progress, setProgress] = useState({ - completed: 0, - total: 0, - }); - - onProgressRef.current = setProgress; - - const { completed, total } = progress; - - return ( - - - - 0 - ? Math.round((completed / total) * 100) - : 0 - } - /> - - {`${completed} / ${total}`} - - - ); -}; - -type ClusterListProps = ClusterResHeaderProps & { - height: number; - width: number; -}; - -const ClusterList: React.FC> = ({ - width, - height, - clusterRes, - children, -}) => { - const [items, setItems] = useState([]); - const listRef = useRef(null); - - const columns = useMemo( - () => Math.max(Math.floor(getFractionFittableColumns(width)), 4), - [width], - ); - - const shrinkRatio = getShrinkRatio(width, columns); - const listItemHeight = 120 * shrinkRatio + 24 + 4; - - useEffect(() => { - setItems(clusterRes ? itemsFromClusterRes(clusterRes, columns) : []); - }, [columns, clusterRes]); - - useEffect(() => { - listRef.current?.resetAfterIndex(0); - }, [items]); - - const itemSize = (index: number) => - index === 0 - ? 140 - : index === 1 - ? 110 - : Array.isArray(items[index - 2]) - ? listItemHeight - : 36; - - return ( - - {ClusterListItemRenderer} - - ); -}; - -type Item = string | FaceWithFile[]; - -const itemsFromClusterRes = ( - clusterRes: ClusterDebugPageContents, - columns: number, -) => { - const { clusterPreviewsWithFile, unclusteredFacesWithFile } = clusterRes; - - const result: Item[] = []; - for (let index = 0; index < clusterPreviewsWithFile.length; index++) { - const { clusterSize, faces } = clusterPreviewsWithFile[index]; - result.push(`cluster size ${clusterSize.toFixed(2)}`); - let lastIndex = 0; - while (lastIndex < faces.length) { - result.push(faces.slice(lastIndex, lastIndex + columns)); - lastIndex += columns; - } - } - - if (unclusteredFacesWithFile.length) { - result.push(`•• unclustered faces ${unclusteredFacesWithFile.length}`); - let lastIndex = 0; - while (lastIndex < unclusteredFacesWithFile.length) { - result.push( - unclusteredFacesWithFile.slice(lastIndex, lastIndex + columns), - ); - lastIndex += columns; - } - } - - return result; -}; - -const getFractionFittableColumns = (width: number) => - (width - 2 * getGapFromScreenEdge(width) + 4) / (120 + 4); - -const getGapFromScreenEdge = (width: number) => (width > 4 * 120 ? 24 : 4); - -const getShrinkRatio = (width: number, columns: number) => - (width - 2 * getGapFromScreenEdge(width) - (columns - 1) * 4) / - (columns * 120); - -// It in necessary to define the item renderer otherwise it gets recreated every -// time the parent rerenders, causing the form to lose its submitting state. -const ClusterListItemRenderer = React.memo( - ({ index, style, data }) => { - const { clusterRes, columns, shrinkRatio, items, children } = data; - - if (index == 0) return
{children}
; - - if (index == 1) - return ( -
- -
- ); - - const item = items[index - 2]; - return ( - - - {!Array.isArray(item) ? ( - {item} - ) : ( - item.map((f, i) => ( - - )) - )} - - - ); - }, - areEqual, -); - -interface ClusterResHeaderProps { - clusterRes: ClusterDebugPageContents | undefined; -} - -const ClusterResHeader: React.FC = ({ clusterRes }) => { - if (!clusterRes) return null; - - const { - totalFaceCount, - filteredFaceCount, - clusteredFaceCount, - unclusteredFaceCount, - timeTakenMs, - clusters, - } = clusterRes; - - return ( - - - {`${clusters.length} clusters in ${(timeTakenMs / 1000).toFixed(0)} seconds • ${totalFaceCount} faces ${filteredFaceCount} filtered ${clusteredFaceCount} clustered ${unclusteredFaceCount} unclustered`} - - - Showing only top 30 clusters, bottom 30 clusters, and - unclustered faces. - - - For each cluster showing only up to 50 faces, sorted by cosine - similarity to its highest scoring face. - - - Below each face is its blur, score, cosineSimilarity, direction. - Bad faces are outlined. - - - ); -}; - -const ListItem = styled("div")` - display: flex; - justify-content: center; -`; - -const ListContainer = styled(Box, { - shouldForwardProp: (propName) => propName != "shrinkRatio", -})<{ - columns: number; - shrinkRatio: number; -}>` - display: grid; - grid-template-columns: ${({ columns, shrinkRatio }) => - `repeat(${columns},${120 * shrinkRatio}px)`}; - grid-column-gap: 4px; - width: 100%; - padding: 4px; -`; - -const ListItemContainer = styled(FlexWrapper)<{ span: number }>` - grid-column: span ${(props) => props.span}; -`; - -const LabelContainer = styled(ListItemContainer)` - color: ${({ theme }) => theme.colors.text.muted}; - height: 32px; -`; - -interface FaceItemProps { - faceWithFile: FaceWithFile; -} - -interface FaceWithFile { - face: ClusterFace; - enteFile: EnteFile; - cosineSimilarity?: number; - wasMerged?: boolean; -} - -const FaceItem: React.FC = ({ faceWithFile }) => { - const { face, enteFile, cosineSimilarity } = faceWithFile; - const { faceID, isBadFace } = face; - - const [objectURL, setObjectURL] = useState(); - - useEffect(() => { - let didCancel = false; - let thisObjectURL: string | undefined; - - void faceCrop(faceID, enteFile).then((blob) => { - if (blob && !didCancel) - setObjectURL((thisObjectURL = URL.createObjectURL(blob))); - }); - - return () => { - didCancel = true; - if (thisObjectURL) URL.revokeObjectURL(thisObjectURL); - }; - }, [faceID, enteFile]); - - const fd = faceDirection(face.detection); - const d = fd == "straight" ? "•" : fd == "left" ? "←" : "→"; - return ( - - {objectURL && ( - - )} - - - {`b${face.blur.toFixed(0)} `} - - - {`s${face.score.toFixed(1)}`} - - {cosineSimilarity && ( - - {`c${cosineSimilarity.toFixed(1)}`} - - )} - - {`d${d}`} - - - - ); -}; - -const FaceChip = styled(Box)` - width: 120px; - height: 120px; -`; diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index dde90b5368..3c3c635c1c 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -8,9 +8,11 @@ import { enableML, mlStatusSnapshot, mlStatusSubscribe, + wipClusterDebugPageContents, wipClusterEnable, type MLStatus, } from "@/new/photos/services/ml"; +import { type ClusteringProgress } from "@/new/photos/services/ml/cluster"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import { EnteMenuItem } from "@ente/shared/components/Menu/EnteMenuItem"; import { @@ -20,6 +22,7 @@ import { Divider, FormControlLabel, FormGroup, + LinearProgress, Link, Paper, Stack, @@ -27,7 +30,6 @@ import { type DialogProps, } from "@mui/material"; import { t } from "i18next"; -import { useRouter } from "next/router"; import React, { useEffect, useState, useSyncExternalStore } from "react"; import { Trans } from "react-i18next"; import type { NewAppContextPhotos } from "../types/context"; @@ -300,6 +302,7 @@ const ManageML: React.FC = ({ }) => { const [showClusterOpt, setShowClusterOpt] = useState(false); const { phase, nSyncedFiles, nTotalFiles } = mlStatus; + const [progress, setProgress] = useState(); useEffect(() => void wipClusterEnable().then(setShowClusterOpt), []); @@ -339,8 +342,10 @@ const ManageML: React.FC = ({ }; // TODO-Cluster - const router = useRouter(); - const wipClusterDebug = () => router.push("/cluster-debug"); + const wipClusterDebug = async () => { + await wipClusterDebugPageContents(setProgress); + setProgress(undefined); + }; return ( @@ -400,6 +405,22 @@ const ManageML: React.FC = ({ "Create and show in-memory clusters (not saved or synced). You can also view them in the search dropdown later.", )} /> + {progress && ( + + 0 + ? Math.round( + (progress.completed / + progress.total) * + 100, + ) + : 0 + } + /> + + )} )} diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 6fcecedba2..8d791c7fc0 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -26,16 +26,16 @@ export interface FaceCluster { faces: string[]; } -export interface ClusteringOpts { - minBlur: number; - minScore: number; - minClusterSize: number; - joinThreshold: number; - earlyExitThreshold: number; - batchSize: number; - offsetIncrement: number; - badFaceHeuristics: boolean; -} +const clusteringOptions = { + minBlur: 10, + minScore: 0.8, + minClusterSize: 2, + joinThreshold: 0.76, + earlyExitThreshold: 0.9, + batchSize: 10000, + offsetIncrement: 7500, + badFaceHeuristics: true, +}; export interface ClusteringProgress { completed: number; @@ -69,7 +69,6 @@ export interface ClusterPreviewFace { export const clusterFaces = ( faceIndexes: FaceIndex[], localFiles: EnteFile[], - opts: ClusteringOpts, onProgress: OnClusteringProgress, ) => { const { @@ -81,7 +80,7 @@ export const clusterFaces = ( batchSize, offsetIncrement, badFaceHeuristics, - } = opts; + } = clusteringOptions; const t = Date.now(); const localFileByID = new Map(localFiles.map((f) => [f.id, f])); @@ -223,7 +222,7 @@ export const clusterFaces = ( const timeTakenMs = Date.now() - t; log.info( - `Clustered ${faces.length} faces into ${sortedClusters.length} clusters, ${faces.length - clusterIDForFaceID.size} faces remain unclustered (${timeTakenMs} ms)`, + `Generated ${sortedClusters.length} clusters from ${totalFaceCount} faces (${filteredFaceCount} filtered ${clusteredFaceCount} clustered ${unclusteredFaceCount} unclustered) (${timeTakenMs} ms)`, ); return { diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 1b106123d6..5af8e3b240 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -22,7 +22,6 @@ import type { UploadItem } from "../upload/types"; import { syncCGroups, updatedPeople, type Person } from "./cgroups"; import { type ClusterFace, - type ClusteringOpts, type ClusterPreviewFace, type FaceCluster, type OnClusteringProgress, @@ -378,12 +377,10 @@ export interface ClusterDebugPageContents { } export const wipClusterDebugPageContents = async ( - opts: ClusteringOpts, onProgress: OnClusteringProgress, ): Promise => { if (!(await wipClusterEnable())) throw new Error("Not implemented"); - log.info("clustering", opts); _wip_isClustering = true; _wip_peopleLocal = undefined; triggerStatusUpdate(); @@ -395,7 +392,7 @@ export const wipClusterDebugPageContents = async ( cgroups, unclusteredFaces, ...rest - } = await worker().then((w) => w.clusterFaces(opts, proxy(onProgress))); + } = await worker().then((w) => w.clusterFaces(proxy(onProgress))); const fileForFace = ({ faceID }: { faceID: string }) => ensure(localFileByID.get(ensure(fileIDFromFaceID(faceID)))); diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index ba097e9054..e668ed070c 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -25,11 +25,7 @@ import { indexCLIP, type CLIPIndex, } from "./clip"; -import { - clusterFaces, - type ClusteringOpts, - type OnClusteringProgress, -} from "./cluster"; +import { clusterFaces, type OnClusteringProgress } from "./cluster"; import { saveFaceCrops } from "./crop"; import { getFaceIndexes, @@ -281,11 +277,10 @@ export class MLWorker { } // TODO-Cluster - async clusterFaces(opts: ClusteringOpts, onProgress: OnClusteringProgress) { + async clusterFaces(onProgress: OnClusteringProgress) { return clusterFaces( await getFaceIndexes(), await getAllLocalFiles(), - opts, onProgress, ); } From 5d6ac29d71ccb04bb9e4494a3fd646c32b912609 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 12:10:39 +0530 Subject: [PATCH 02/16] Remove no-longer used hdbscan code We'll follow mobile's linear clustering. --- web/apps/photos/package.json | 1 - web/docs/dependencies.md | 3 -- .../new/photos/services/ml/cluster-hdb.ts | 35 ------------------- web/yarn.lock | 12 ------- 4 files changed, 51 deletions(-) delete mode 100644 web/packages/new/photos/services/ml/cluster-hdb.ts diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index f7a6b93f09..59f3c3b7dd 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -17,7 +17,6 @@ "exifreader": "^4", "fast-srp-hap": "^2.0.4", "ffmpeg-wasm": "file:./thirdparty/ffmpeg-wasm", - "hdbscan": "0.0.1-alpha.5", "leaflet": "^1.9.4", "leaflet-defaulticon-compatibility": "^0.1.1", "localforage": "^1.9.0", diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index 7451f802bb..b8e13fe8c7 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -209,9 +209,6 @@ For more details, see [translations.md](translations.md). > provides affine transforms, while `matrix` is for performing computations > on matrices, say inverting them or performing their decomposition. -- [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face - clustering. - ## Auth app specific - [otpauth](https://github.com/hectorm/otpauth) is used for the generation of diff --git a/web/packages/new/photos/services/ml/cluster-hdb.ts b/web/packages/new/photos/services/ml/cluster-hdb.ts deleted file mode 100644 index 3ecda4b5bc..0000000000 --- a/web/packages/new/photos/services/ml/cluster-hdb.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Hdbscan, type DebugInfo } from "hdbscan"; - -/** - * Each "cluster" is a list of indexes of the embeddings belonging to that - * particular cluster. - */ -export type EmbeddingCluster = number[]; - -export interface ClusterHdbscanResult { - clusters: EmbeddingCluster[]; - noise: number[]; - debugInfo?: DebugInfo; -} - -/** - * Cluster the given {@link embeddings} using hdbscan. - */ -export const clusterHdbscan = ( - embeddings: number[][], -): ClusterHdbscanResult => { - const hdbscan = new Hdbscan({ - input: embeddings, - minClusterSize: 3, - minSamples: 5, - clusterSelectionEpsilon: 0.6, - clusterSelectionMethod: "leaf", - debug: false, - }); - - return { - clusters: hdbscan.getClusters(), - noise: hdbscan.getNoise(), - debugInfo: hdbscan.getDebugInfo(), - }; -}; diff --git a/web/yarn.lock b/web/yarn.lock index eb8911c489..2d6425cb67 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -2671,13 +2671,6 @@ hasown@^2.0.0, hasown@^2.0.1, hasown@^2.0.2: dependencies: function-bind "^1.1.2" -hdbscan@0.0.1-alpha.5: - version "0.0.1-alpha.5" - resolved "https://registry.yarnpkg.com/hdbscan/-/hdbscan-0.0.1-alpha.5.tgz#8b0cd45243fa60d2fe83e31f1e8bc939ff374c0d" - integrity sha512-Jv92UaFFRAMcK8GKhyxlSGvkf5pf9Y9HpmRQyyWfWop5nm2zs2NmgGG3wOCYo5zy1AeZFtVJjgbpaPjR0IsR/Q== - dependencies: - kd-tree-javascript "^1.0.3" - heic-convert@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/heic-convert/-/heic-convert-2.1.0.tgz#7f764529e37591ae263ef49582d1d0c13491526e" @@ -3121,11 +3114,6 @@ jszip@^3.10.1: readable-stream "~2.3.6" setimmediate "^1.0.5" -kd-tree-javascript@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/kd-tree-javascript/-/kd-tree-javascript-1.0.3.tgz#ab5239ed44e347e10065590fd479e947bedff96c" - integrity sha512-7oSugmaxTCJFqey11rlTSEQD3hGDnRgROMj9MEREvDGV8SlIFwN7x3jJRyFoi+mjO0+4wuSuaDLS1reNQHP7uA== - keyv@^4.5.3: version "4.5.4" resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" From 7b552a1ee347f2ae61ecf67635c760a64baba48b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 12:17:40 +0530 Subject: [PATCH 03/16] count --- web/packages/new/photos/components/PeopleList.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/components/PeopleList.tsx b/web/packages/new/photos/components/PeopleList.tsx index 271ec2e31c..a5cc7d1ee0 100644 --- a/web/packages/new/photos/components/PeopleList.tsx +++ b/web/packages/new/photos/components/PeopleList.tsx @@ -20,7 +20,9 @@ export const SearchPeopleList: React.FC = ({ }) => { const isMobileWidth = useIsMobileWidth(); return ( - + 3 ? "center" : "start" }} + > {people.slice(0, isMobileWidth ? 6 : 7).map((person) => ( = ({ const SearchPeopleContainer = styled("div")` display: flex; flex-wrap: wrap; - justify-content: center; align-items: center; gap: 5px; margin-block: 12px; From e8b692b5ad60298c0557d32f8ca7c7926811d405 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 12:29:42 +0530 Subject: [PATCH 04/16] Prep for clustering updates --- web/packages/new/photos/services/ml/index.ts | 6 ++---- web/packages/new/photos/services/ml/worker-types.ts | 11 ++++++----- web/packages/new/photos/services/ml/worker.ts | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 5af8e3b240..0f6b32aed3 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -100,9 +100,7 @@ const worker = () => const createComlinkWorker = async () => { const electron = ensureElectron(); - const delegate = { - workerDidProcessFileOrIdle, - }; + const delegate = { workerDidUpdateStatus }; // Obtain a message port from the Electron layer. const messagePort = await createMLWorker(electron); @@ -593,7 +591,7 @@ const setInterimScheduledStatus = () => { setMLStatusSnapshot({ phase: "scheduled", nSyncedFiles, nTotalFiles }); }; -const workerDidProcessFileOrIdle = throttled(updateMLStatusSnapshot, 2000); +const workerDidUpdateStatus = throttled(updateMLStatusSnapshot, 2000); /** * A function that can be used to subscribe to updates to {@link Person}s. diff --git a/web/packages/new/photos/services/ml/worker-types.ts b/web/packages/new/photos/services/ml/worker-types.ts index 446986b8ef..b12598d52b 100644 --- a/web/packages/new/photos/services/ml/worker-types.ts +++ b/web/packages/new/photos/services/ml/worker-types.ts @@ -3,15 +3,16 @@ */ /** - * Callbacks invoked by the worker at various points in the indexing pipeline to - * notify the main thread of events it might be interested in. + * Callbacks invoked by the worker at various points in the indexing and + * clustering pipeline to notify the main thread of events it might be + * interested in. */ export interface MLWorkerDelegate { /** - * Called whenever the worker processes a file during indexing (either - * successfully or with errors), or when in goes into the "idle" state. + * Called whenever the worker does some action that might need the UI state + * indicating the indexing or clustering status to be updated. */ - workerDidProcessFileOrIdle: () => void; + workerDidUpdateStatus: () => void; } /** diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index e668ed070c..e43af4ccea 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -242,7 +242,7 @@ export class MLWorker { this.state = "idle"; this.idleDuration = Math.min(this.idleDuration * 2, idleDurationMax); this.idleTimeout = setTimeout(scheduleTick, this.idleDuration * 1000); - this.delegate?.workerDidProcessFileOrIdle(); + this.delegate?.workerDidUpdateStatus(); } /** Return the next batch of items to backfill (if any). */ @@ -342,7 +342,7 @@ const indexNextBatch = async ( await Promise.race(tasks); // Let the main thread now we're doing something. - delegate?.workerDidProcessFileOrIdle(); + delegate?.workerDidUpdateStatus(); // Let us drain the microtask queue. This also gives a chance for other // interactive tasks like `clipMatches` to run. From c4f70c370ecfe9a2b2dfbc736d8caf553ad47494 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 13:04:39 +0530 Subject: [PATCH 05/16] Integrate clustering progress into ML status --- .../new/photos/components/MLSettings.tsx | 31 ++----------------- .../new/photos/components/SearchBar.tsx | 1 + .../new/photos/services/ml/cluster.ts | 4 +-- web/packages/new/photos/services/ml/index.ts | 27 +++++++++------- web/packages/new/photos/services/ml/worker.ts | 22 ++++++++++--- 5 files changed, 38 insertions(+), 47 deletions(-) diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index 3c3c635c1c..43017e8eca 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -8,11 +8,10 @@ import { enableML, mlStatusSnapshot, mlStatusSubscribe, - wipClusterDebugPageContents, + wipCluster, wipClusterEnable, type MLStatus, } from "@/new/photos/services/ml"; -import { type ClusteringProgress } from "@/new/photos/services/ml/cluster"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import { EnteMenuItem } from "@ente/shared/components/Menu/EnteMenuItem"; import { @@ -22,7 +21,6 @@ import { Divider, FormControlLabel, FormGroup, - LinearProgress, Link, Paper, Stack, @@ -302,7 +300,6 @@ const ManageML: React.FC = ({ }) => { const [showClusterOpt, setShowClusterOpt] = useState(false); const { phase, nSyncedFiles, nTotalFiles } = mlStatus; - const [progress, setProgress] = useState(); useEffect(() => void wipClusterEnable().then(setShowClusterOpt), []); @@ -341,12 +338,6 @@ const ManageML: React.FC = ({ }); }; - // TODO-Cluster - const wipClusterDebug = async () => { - await wipClusterDebugPageContents(setProgress); - setProgress(undefined); - }; - return ( @@ -397,30 +388,14 @@ const ManageML: React.FC = ({ label={ut( "Create clusters • internal only option", )} - onClick={wipClusterDebug} + onClick={() => void wipCluster()} /> - {progress && ( - - 0 - ? Math.round( - (progress.completed / - progress.total) * - 100, - ) - : 0 - } - /> - - )} )} diff --git a/web/packages/new/photos/components/SearchBar.tsx b/web/packages/new/photos/components/SearchBar.tsx index 3ad465e26a..e46b4d3cc0 100644 --- a/web/packages/new/photos/components/SearchBar.tsx +++ b/web/packages/new/photos/components/SearchBar.tsx @@ -419,6 +419,7 @@ const EmptyState: React.FC = ({ label = t("indexing_fetching", mlStatus); break; case "clustering": + // TODO-Cluster label = t("indexing_people", mlStatus); break; case "done": diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 8d791c7fc0..4907f53911 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -42,8 +42,6 @@ export interface ClusteringProgress { total: number; } -export type OnClusteringProgress = (progress: ClusteringProgress) => void; - /** A {@link Face} annotated with data needed during clustering. */ export type ClusterFace = Omit & { embedding: Float32Array; @@ -69,7 +67,7 @@ export interface ClusterPreviewFace { export const clusterFaces = ( faceIndexes: FaceIndex[], localFiles: EnteFile[], - onProgress: OnClusteringProgress, + onProgress: (progress: ClusteringProgress) => void, ) => { const { minBlur, diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 0f6b32aed3..225c0891c6 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -24,7 +24,6 @@ import { type ClusterFace, type ClusterPreviewFace, type FaceCluster, - type OnClusteringProgress, } from "./cluster"; import { regenerateFaceCrops } from "./crop"; import { clearMLDB, getFaceIndex, getIndexableAndIndexedCounts } from "./db"; @@ -340,7 +339,6 @@ export const wipClusterEnable = async (): Promise => (await isInternalUser()); // // TODO-Cluster temporary state here -let _wip_isClustering = false; let _wip_peopleLocal: Person[] | undefined; let _wip_peopleRemote: Person[] | undefined; let _wip_hasSwitchedOnce = false; @@ -374,12 +372,9 @@ export interface ClusterDebugPageContents { }[]; } -export const wipClusterDebugPageContents = async ( - onProgress: OnClusteringProgress, -): Promise => { +export const wipCluster = async () => { if (!(await wipClusterEnable())) throw new Error("Not implemented"); - _wip_isClustering = true; _wip_peopleLocal = undefined; triggerStatusUpdate(); @@ -390,7 +385,7 @@ export const wipClusterDebugPageContents = async ( cgroups, unclusteredFaces, ...rest - } = await worker().then((w) => w.clusterFaces(proxy(onProgress))); + } = await worker().then((w) => w.clusterFaces()); const fileForFace = ({ faceID }: { faceID: string }) => ensure(localFileByID.get(ensure(fileIDFromFaceID(faceID)))); @@ -441,7 +436,6 @@ export const wipClusterDebugPageContents = async ( .filter((c) => !!c) .sort((a, b) => b.faceIDs.length - a.faceIDs.length); - _wip_isClustering = false; _wip_peopleLocal = people; triggerStatusUpdate(); setPeopleSnapshot((_wip_peopleRemote ?? []).concat(people)); @@ -540,6 +534,19 @@ const setMLStatusSnapshot = (snapshot: MLStatus) => { const getMLStatus = async (): Promise => { if (!_state.isMLEnabled) return { phase: "disabled" }; + const w = await worker(); + + // The worker has a clustering progress set iff it is clustering. This + // overrides other behaviours. + const clusteringProgress = await w.clusteringProgess; + if (clusteringProgress) { + return { + phase: "clustering", + nSyncedFiles: clusteringProgress.completed, + nTotalFiles: clusteringProgress.total, + }; + } + const { indexedCount, indexableCount } = await getIndexableAndIndexedCounts(); @@ -551,11 +558,9 @@ const getMLStatus = async (): Promise => { // indexable count. let phase: MLStatus["phase"]; - const state = await (await worker()).state; + const state = await w.state; if (state == "indexing" || state == "fetching") { phase = state; - } else if (_wip_isClustering) { - phase = "clustering"; } else if (state == "init" || indexableCount > 0) { phase = "scheduled"; } else { diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index e43af4ccea..5f93257bad 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -25,7 +25,7 @@ import { indexCLIP, type CLIPIndex, } from "./clip"; -import { clusterFaces, type OnClusteringProgress } from "./cluster"; +import { clusterFaces, type ClusteringProgress } from "./cluster"; import { saveFaceCrops } from "./crop"; import { getFaceIndexes, @@ -97,6 +97,8 @@ interface IndexableItem { export class MLWorker { /** The last known state of the worker. */ public state: WorkerState = "init"; + /** If the worker is currently clustering, then its last known progress. */ + public clusteringProgess: ClusteringProgress | undefined; private electron: ElectronMLWorker | undefined; private delegate: MLWorkerDelegate | undefined; @@ -276,13 +278,23 @@ export class MLWorker { })); } - // TODO-Cluster - async clusterFaces(onProgress: OnClusteringProgress) { - return clusterFaces( + /** + * Run face clustering on all faces. + * + * This should only be invoked when the face indexing (including syncing + * with remote) is complete so that we cluster the latest set of faces. + */ + async clusterFaces() { + const result = clusterFaces( await getFaceIndexes(), await getAllLocalFiles(), - onProgress, + (progress) => { + this.clusteringProgess = progress; + this.delegate?.workerDidUpdateStatus(); + }, ); + this.clusteringProgess = undefined; + return result; } } From 345cc2f34fca5f807c509c41250b34c168163005 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 13:43:04 +0530 Subject: [PATCH 06/16] Fix the UI updates --- .../new/photos/services/ml/cluster.ts | 20 +++++++++++++++---- web/packages/new/photos/services/ml/worker.ts | 14 +++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index 4907f53911..d002fd770f 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -2,6 +2,7 @@ import { assertionFailed } from "@/base/assert"; import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; +import { wait } from "@/utils/promise"; import type { EnteFile } from "../../types/file"; import type { AnnotatedCGroup } from "./cgroups"; import { faceDirection, type Face, type FaceIndex } from "./face"; @@ -63,8 +64,15 @@ export interface ClusterPreviewFace { * Generates clusters from the given faces using a batched form of linear * clustering, with a bit of lookback (and a dollop of heuristics) to get the * clusters to merge across batches. + * + * [Note: Draining the event loop during clustering] + * + * The clustering is a synchronous operation, but we make it async to + * artificially drain the worker's event loop after each mini-batch so that + * other interactions with the worker (where this code runs) do not get stalled + * while clustering is in progress. */ -export const clusterFaces = ( +export const clusterFaces = async ( faceIndexes: FaceIndex[], localFiles: EnteFile[], onProgress: (progress: ClusteringProgress) => void, @@ -134,7 +142,7 @@ export const clusterFaces = ( clusters, }; - const newState = clusterBatchLinear( + const newState = await clusterBatchLinear( batch, oldState, joinThreshold, @@ -307,7 +315,7 @@ interface ClusteringState { clusters: FaceCluster[]; } -const clusterBatchLinear = ( +const clusterBatchLinear = async ( faces: ClusterFace[], oldState: ClusteringState, joinThreshold: number, @@ -328,7 +336,11 @@ const clusterBatchLinear = ( // For each face in the batch for (const [i, fi] of faces.entries()) { - if (i % 100 == 0) onProgress({ completed: i, total: faces.length }); + if (i % 100 == 0) { + onProgress({ completed: i, total: faces.length }); + // See: [Note: Draining the event loop during clustering] + await wait(0); + } // If the face is already part of a cluster, then skip it. if (state.clusterIDForFaceID.has(fi.faceID)) continue; diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 5f93257bad..613aacb632 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -285,17 +285,19 @@ export class MLWorker { * with remote) is complete so that we cluster the latest set of faces. */ async clusterFaces() { - const result = clusterFaces( + const result = await clusterFaces( await getFaceIndexes(), await getAllLocalFiles(), - (progress) => { - this.clusteringProgess = progress; - this.delegate?.workerDidUpdateStatus(); - }, + (progress) => this.updateClusteringProgress(progress), ); - this.clusteringProgess = undefined; + this.updateClusteringProgress(undefined); return result; } + + private updateClusteringProgress(progress: ClusteringProgress | undefined) { + this.clusteringProgess = progress; + this.delegate?.workerDidUpdateStatus(); + } } expose(MLWorker); From 8ca3b80e949b4359522c8ea59898a2553ebfd7c1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 13:52:24 +0530 Subject: [PATCH 07/16] Match the (temp) search placeholder message --- web/packages/new/photos/components/MLSettings.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index 43017e8eca..6158387a80 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -1,7 +1,7 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { MenuItemGroup, MenuSectionTitle } from "@/base/components/Menu"; import { Titlebar } from "@/base/components/Titlebar"; -import { pt, ut } from "@/base/i18n"; +import { ut } from "@/base/i18n"; import log from "@/base/log"; import { disableML, @@ -316,7 +316,7 @@ const ManageML: React.FC = ({ break; case "clustering": // TODO-Cluster - status = pt("Grouping faces"); + status = t("people"); break; default: status = t("indexing_status_done"); From df17b11573a8810e9857f49dcb90b917cfbeef7c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 15:33:01 +0530 Subject: [PATCH 08/16] Make the animation fit the page better --- web/packages/new/photos/components/PeopleList.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/packages/new/photos/components/PeopleList.tsx b/web/packages/new/photos/components/PeopleList.tsx index a5cc7d1ee0..8d77b737c6 100644 --- a/web/packages/new/photos/components/PeopleList.tsx +++ b/web/packages/new/photos/components/PeopleList.tsx @@ -196,6 +196,10 @@ const FaceCropImageView: React.FC = ({ ) : ( theme.colors.background.elevated2, + }} width={placeholderDimension} height={placeholderDimension} /> From 18a0b18a13262534a78336a58c0c49a74c3955ae Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 15:47:40 +0530 Subject: [PATCH 09/16] Auto debug --- web/apps/photos/src/pages/gallery.tsx | 12 ++---------- web/packages/new/photos/services/ml/index.ts | 7 ++++--- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 121c10e5e8..216789f9b9 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -14,7 +14,7 @@ import { getLocalFiles, getLocalTrashedFiles, } from "@/new/photos/services/files"; -import { wipHasSwitchedOnceCmpAndSet } from "@/new/photos/services/ml"; +import { wipClusterLocalOnce } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/cgroups"; import { filterSearchableFiles, @@ -681,15 +681,7 @@ export default function Gallery() { }; }, [selectAll, clearSelection]); - useEffect(() => { - // TODO-Cluster - if (process.env.NEXT_PUBLIC_ENTE_WIP_CL_AUTO) { - setTimeout(() => { - if (!wipHasSwitchedOnceCmpAndSet()) - router.push("cluster-debug"); - }, 2000); - } - }, []); + useEffect(() => wipClusterLocalOnce(), []); const fileToCollectionsMap = useMemoSingleThreaded(() => { return constructFileToCollectionMap(files); diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 225c0891c6..a1aa1790c8 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -343,10 +343,11 @@ let _wip_peopleLocal: Person[] | undefined; let _wip_peopleRemote: Person[] | undefined; let _wip_hasSwitchedOnce = false; -export const wipHasSwitchedOnceCmpAndSet = () => { - if (_wip_hasSwitchedOnce) return true; +export const wipClusterLocalOnce = () => { + if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL_AUTO) return; + if (_wip_hasSwitchedOnce) return; _wip_hasSwitchedOnce = true; - return false; + void wipCluster(); }; export interface ClusterPreviewWithFile { From 6344a3c6402a2773b456e49dc1dd0b4dfde1bb8b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 15:55:14 +0530 Subject: [PATCH 10/16] bona fide --- web/packages/new/photos/services/ml/index.ts | 29 +++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index a1aa1790c8..d156cbbeca 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -83,6 +83,20 @@ class MLState { */ peopleSnapshot: Person[] | undefined; + /** + * Cached in-memory copy of people generated from local clusters. + * + * Part of {@link peopleSnapshot}. + */ + peopleLocal: Person[] = []; + + /** + * Cached in-memory copy of people generated from remote cgroups. + * + * Part of {@link peopleSnapshot}. + */ + peopleRemote: Person[] = []; + /** * In flight face crop regeneration promises indexed by the IDs of the files * whose faces we are regenerating. @@ -339,8 +353,6 @@ export const wipClusterEnable = async (): Promise => (await isInternalUser()); // // TODO-Cluster temporary state here -let _wip_peopleLocal: Person[] | undefined; -let _wip_peopleRemote: Person[] | undefined; let _wip_hasSwitchedOnce = false; export const wipClusterLocalOnce = () => { @@ -376,7 +388,7 @@ export interface ClusterDebugPageContents { export const wipCluster = async () => { if (!(await wipClusterEnable())) throw new Error("Not implemented"); - _wip_peopleLocal = undefined; + _state.peopleLocal = []; triggerStatusUpdate(); const { @@ -437,9 +449,9 @@ export const wipCluster = async () => { .filter((c) => !!c) .sort((a, b) => b.faceIDs.length - a.faceIDs.length); - _wip_peopleLocal = people; + _state.peopleLocal = people; triggerStatusUpdate(); - setPeopleSnapshot((_wip_peopleRemote ?? []).concat(people)); + updatePeopleSnapshot(); return { clusters, @@ -632,6 +644,9 @@ export const peopleSubscribe = (onChange: () => void): (() => void) => { */ export const peopleSnapshot = () => _state.peopleSnapshot; +const updatePeopleSnapshot = () => + setPeopleSnapshot(_state.peopleRemote.concat(_state.peopleLocal)); + const setPeopleSnapshot = (snapshot: Person[] | undefined) => { _state.peopleSnapshot = snapshot; _state.peopleListeners.forEach((l) => l()); @@ -643,8 +658,8 @@ const setPeopleSnapshot = (snapshot: Person[] | undefined) => { */ const updatePeople = async () => { const people = await updatedPeople(); - _wip_peopleRemote = people; - setPeopleSnapshot(people.concat(_wip_peopleLocal ?? [])); + _state.peopleRemote = people; + updatePeopleSnapshot(); setSearchPeople(people); }; From 000fe87ebb188439d92b3562c0e7a9381ac07bc8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 16:00:59 +0530 Subject: [PATCH 11/16] Prune unused --- .../new/photos/services/ml/cluster.ts | 33 ---------- web/packages/new/photos/services/ml/index.ts | 63 +------------------ 2 files changed, 2 insertions(+), 94 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index d002fd770f..c323d8ad2f 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -173,28 +173,6 @@ export const clusterFaces = async ( (a, b) => b.faces.length - a.faces.length, ); - // Convert into the data structure we're using to debug/visualize. - const clusterPreviewClusters = - sortedClusters.length < 60 - ? sortedClusters - : sortedClusters.slice(0, 30).concat(sortedClusters.slice(-30)); - const clusterPreviews = clusterPreviewClusters.map((cluster) => { - const faces = cluster.faces.map((id) => ensure(faceForFaceID.get(id))); - const topFace = faces.reduce((top, face) => - top.score > face.score ? top : face, - ); - const previewFaces: ClusterPreviewFace[] = faces.map((face) => { - const csim = dotProduct(topFace.embedding, face.embedding); - return { face, cosineSimilarity: csim, wasMerged: false }; - }); - return { - clusterSize: cluster.faces.length, - faces: previewFaces - .sort((a, b) => b.cosineSimilarity - a.cosineSimilarity) - .slice(0, 50), - }; - }); - // TODO-Cluster - Currently we're not syncing with remote or saving anything // locally, so cgroups will be empty. Create a temporary (unsaved, unsynced) // cgroup, one per cluster. @@ -222,26 +200,15 @@ export const clusterFaces = async ( const clusteredFaceCount = clusterIDForFaceID.size; const unclusteredFaceCount = filteredFaceCount - clusteredFaceCount; - const unclusteredFaces = faces.filter( - ({ faceID }) => !clusterIDForFaceID.has(faceID), - ); - const timeTakenMs = Date.now() - t; log.info( `Generated ${sortedClusters.length} clusters from ${totalFaceCount} faces (${filteredFaceCount} filtered ${clusteredFaceCount} clustered ${unclusteredFaceCount} unclustered) (${timeTakenMs} ms)`, ); return { - totalFaceCount, - filteredFaceCount, - clusteredFaceCount, - unclusteredFaceCount, localFileByID, - clusterPreviews, clusters: sortedClusters, cgroups, - unclusteredFaces: unclusteredFaces, - timeTakenMs, }; }; diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index d156cbbeca..19a66590a6 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -20,11 +20,6 @@ import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import { setSearchPeople } from "../search"; import type { UploadItem } from "../upload/types"; import { syncCGroups, updatedPeople, type Person } from "./cgroups"; -import { - type ClusterFace, - type ClusterPreviewFace, - type FaceCluster, -} from "./cluster"; import { regenerateFaceCrops } from "./crop"; import { clearMLDB, getFaceIndex, getIndexableAndIndexedCounts } from "./db"; import { MLWorker } from "./worker"; @@ -362,63 +357,16 @@ export const wipClusterLocalOnce = () => { void wipCluster(); }; -export interface ClusterPreviewWithFile { - clusterSize: number; - faces: ClusterPreviewFaceWithFile[]; -} - -export type ClusterPreviewFaceWithFile = ClusterPreviewFace & { - enteFile: EnteFile; -}; - -export interface ClusterDebugPageContents { - totalFaceCount: number; - filteredFaceCount: number; - clusteredFaceCount: number; - unclusteredFaceCount: number; - timeTakenMs: number; - clusters: FaceCluster[]; - clusterPreviewsWithFile: ClusterPreviewWithFile[]; - unclusteredFacesWithFile: { - face: ClusterFace; - enteFile: EnteFile; - }[]; -} - export const wipCluster = async () => { if (!(await wipClusterEnable())) throw new Error("Not implemented"); _state.peopleLocal = []; triggerStatusUpdate(); - const { - localFileByID, - clusterPreviews, - clusters, - cgroups, - unclusteredFaces, - ...rest - } = await worker().then((w) => w.clusterFaces()); - - const fileForFace = ({ faceID }: { faceID: string }) => - ensure(localFileByID.get(ensure(fileIDFromFaceID(faceID)))); - - const clusterPreviewsWithFile = clusterPreviews.map( - ({ clusterSize, faces }) => ({ - clusterSize, - faces: faces.map(({ face, ...rest }) => ({ - face, - enteFile: fileForFace(face), - ...rest, - })), - }), + const { localFileByID, clusters, cgroups } = await worker().then((w) => + w.clusterFaces(), ); - const unclusteredFacesWithFile = unclusteredFaces.map((face) => ({ - face, - enteFile: fileForFace(face), - })); - const clusterByID = new Map(clusters.map((c) => [c.id, c])); const people = cgroups @@ -452,13 +400,6 @@ export const wipCluster = async () => { _state.peopleLocal = people; triggerStatusUpdate(); updatePeopleSnapshot(); - - return { - clusters, - clusterPreviewsWithFile, - unclusteredFacesWithFile, - ...rest, - }; }; export type MLStatus = From 9d1332bff1fd73c8a156681fe619b67ae17ecc3a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 16:04:22 +0530 Subject: [PATCH 12/16] Prune --- web/packages/new/photos/services/ml/cluster.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index c323d8ad2f..e498ebaba1 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -193,16 +193,10 @@ export const clusterFaces = async ( }); } - // TODO-Cluster the total face count is only needed during debugging - let totalFaceCount = 0; - for (const fi of faceIndexes) totalFaceCount += fi.faces.length; - const filteredFaceCount = faces.length; const clusteredFaceCount = clusterIDForFaceID.size; - const unclusteredFaceCount = filteredFaceCount - clusteredFaceCount; - const timeTakenMs = Date.now() - t; log.info( - `Generated ${sortedClusters.length} clusters from ${totalFaceCount} faces (${filteredFaceCount} filtered ${clusteredFaceCount} clustered ${unclusteredFaceCount} unclustered) (${timeTakenMs} ms)`, + `Generated ${sortedClusters.length} clusters from ${faces.length} faces (${clusteredFaceCount} clustered ${faces.length - clusteredFaceCount} unclustered) (${timeTakenMs} ms)`, ); return { From f64c0dcc867b7a4484f6174ac34478d1f51a8ee9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 16:13:49 +0530 Subject: [PATCH 13/16] Move into worker for now --- .../new/photos/services/ml/cluster.ts | 90 ++++++++++++++----- web/packages/new/photos/services/ml/index.ts | 37 +------- 2 files changed, 68 insertions(+), 59 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index e498ebaba1..ffe631da97 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -173,25 +173,10 @@ export const clusterFaces = async ( (a, b) => b.faces.length - a.faces.length, ); - // TODO-Cluster - Currently we're not syncing with remote or saving anything - // locally, so cgroups will be empty. Create a temporary (unsaved, unsynced) - // cgroup, one per cluster. - - const cgroups: AnnotatedCGroup[] = []; - for (const cluster of sortedClusters) { - const faces = cluster.faces.map((id) => ensure(faceForFaceID.get(id))); - const topFace = faces.reduce((top, face) => - top.score > face.score ? top : face, - ); - cgroups.push({ - id: cluster.id, - name: undefined, - assigned: [cluster], - isHidden: false, - avatarFaceID: undefined, - displayFaceID: topFace.faceID, - }); - } + // TODO-Cluster + // This isn't really part of the clustering, but help the main thread out by + // pre-computing temporary in-memory people, one per cluster. + const people = toPeople(sortedClusters, localFileByID, faceForFaceID); const clusteredFaceCount = clusterIDForFaceID.size; const timeTakenMs = Date.now() - t; @@ -199,11 +184,7 @@ export const clusterFaces = async ( `Generated ${sortedClusters.length} clusters from ${faces.length} faces (${clusteredFaceCount} clustered ${faces.length - clusteredFaceCount} unclustered) (${timeTakenMs} ms)`, ); - return { - localFileByID, - clusters: sortedClusters, - cgroups, - }; + return { clusters: sortedClusters, people }; }; /** @@ -355,3 +336,64 @@ const clusterBatchLinear = async ( return state; }; + +/** + * Construct a {@link Person} object for each cluster. + */ +const toPeople = ( + clusters: FaceCluster[], + localFileByID: Map, + faceForFaceID: Map, +) => { + // TODO-Cluster - Currently we're not syncing with remote or saving anything + // locally, so cgroups will be empty. Create a temporary (unsaved, unsynced) + // cgroup, one per cluster. + + const cgroups: AnnotatedCGroup[] = []; + for (const cluster of clusters) { + const faces = cluster.faces.map((id) => ensure(faceForFaceID.get(id))); + const topFace = faces.reduce((top, face) => + top.score > face.score ? top : face, + ); + cgroups.push({ + id: cluster.id, + name: undefined, + assigned: [cluster], + isHidden: false, + avatarFaceID: undefined, + displayFaceID: topFace.faceID, + }); + } + + const clusterByID = new Map(clusters.map((c) => [c.id, c])); + + const people = cgroups + // TODO-Cluster + .map((cgroup) => ({ ...cgroup, name: cgroup.id })) + .map((cgroup) => { + if (!cgroup.name) return undefined; + const faceID = ensure(cgroup.displayFaceID); + const fileID = ensure(fileIDFromFaceID(faceID)); + const file = ensure(localFileByID.get(fileID)); + + const faceIDs = cgroup.assigned + .map(({ id }) => ensure(clusterByID.get(id))) + .flatMap((cluster) => cluster.faces); + const fileIDs = faceIDs + .map((faceID) => fileIDFromFaceID(faceID)) + .filter((fileID) => fileID !== undefined); + + return { + id: cgroup.id, + name: cgroup.name, + faceIDs, + fileIDs: [...new Set(fileIDs)], + displayFaceID: faceID, + displayFaceFile: file, + }; + }) + .filter((c) => !!c) + .sort((a, b) => b.faceIDs.length - a.faceIDs.length); + + return people; +}; diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 19a66590a6..1d84ae7500 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -360,46 +360,13 @@ export const wipClusterLocalOnce = () => { export const wipCluster = async () => { if (!(await wipClusterEnable())) throw new Error("Not implemented"); - _state.peopleLocal = []; triggerStatusUpdate(); - const { localFileByID, clusters, cgroups } = await worker().then((w) => - w.clusterFaces(), - ); - - const clusterByID = new Map(clusters.map((c) => [c.id, c])); - - const people = cgroups - // TODO-Cluster - .map((cgroup) => ({ ...cgroup, name: cgroup.id })) - .map((cgroup) => { - if (!cgroup.name) return undefined; - const faceID = ensure(cgroup.displayFaceID); - const fileID = ensure(fileIDFromFaceID(faceID)); - const file = ensure(localFileByID.get(fileID)); - - const faceIDs = cgroup.assigned - .map(({ id }) => ensure(clusterByID.get(id))) - .flatMap((cluster) => cluster.faces); - const fileIDs = faceIDs - .map((faceID) => fileIDFromFaceID(faceID)) - .filter((fileID) => fileID !== undefined); - - return { - id: cgroup.id, - name: cgroup.name, - faceIDs, - fileIDs: [...new Set(fileIDs)], - displayFaceID: faceID, - displayFaceFile: file, - }; - }) - .filter((c) => !!c) - .sort((a, b) => b.faceIDs.length - a.faceIDs.length); + const { people } = await worker().then((w) => w.clusterFaces()); _state.peopleLocal = people; - triggerStatusUpdate(); updatePeopleSnapshot(); + triggerStatusUpdate(); }; export type MLStatus = From ed1c9df007adeb8e829ab3dab930bc045865c918 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 16:43:39 +0530 Subject: [PATCH 14/16] Funnel the same way --- .../new/photos/services/ml/cgroups.ts | 36 +++----- .../new/photos/services/ml/cluster.ts | 90 ++++++------------- web/packages/new/photos/services/ml/face.ts | 16 +++- web/packages/new/photos/services/ml/index.ts | 15 ---- 4 files changed, 57 insertions(+), 100 deletions(-) diff --git a/web/packages/new/photos/services/ml/cgroups.ts b/web/packages/new/photos/services/ml/cgroups.ts index d6c3e0fbaa..e7316e11f6 100644 --- a/web/packages/new/photos/services/ml/cgroups.ts +++ b/web/packages/new/photos/services/ml/cgroups.ts @@ -1,10 +1,11 @@ import { masterKeyFromSession } from "@/base/session-store"; -import { fileIDFromFaceID, wipClusterEnable } from "."; +import { wipClusterEnable } from "."; import type { EnteFile } from "../../types/file"; import { getLocalFiles } from "../files"; import { pullCGroups } from "../user-entity"; import type { FaceCluster } from "./cluster"; import { getClusterGroups, getFaceIndexes } from "./db"; +import { fileIDFromFaceID } from "./face"; /** * A cgroup ("cluster group") is a group of clusters (possibly containing just a @@ -78,12 +79,15 @@ export interface CGroup { } /** - * A massaged version of {@link CGroup} suitable for being shown in the UI. + * A massaged version of {@link CGroup} or a {@link FaceCluster} suitable for + * being shown in the UI. + * + * We transform both both remote cluster groups and local-only face clusters + * into the same "person" object that can be shown in the UI. * * The cgroups synced with remote do not directly correspond to "people". - * CGroups represent both positive and negative feedback, where the negations - * are specifically feedback meant so that we do not show the corresponding - * cluster in the UI. + * CGroups represent both positive and negative feedback (i.e, the user does not + * wish a particular cluster group to be shown in the UI). * * So while each person has an underlying cgroups, not all cgroups have a * corresponding person. @@ -95,13 +99,15 @@ export interface CGroup { */ export interface Person { /** - * Nanoid of the underlying {@link CGroup}. + * Nanoid of the underlying {@link CGroup} or {@link FaceCluster}. */ id: string; /** * The name of the person. + * + * This will only be set for named cgroups. */ - name: string; + name: string | undefined; /** * IDs of the (unique) files in which this face occurs. */ @@ -117,22 +123,6 @@ export interface Person { displayFaceFile: EnteFile; } -// TODO-Cluster remove me -/** - * A {@link CGroup} annotated with various in-memory state to make it easier for - * the upper layers of our code to directly use it. - */ -export type AnnotatedCGroup = CGroup & { - /** - * Locally determined ID of the "best" face that should be used as the - * display face, to represent this cluster group in the UI. - * - * This property is not synced with remote. For more details, see - * {@link avatarFaceID}. - */ - displayFaceID: string | undefined; -}; - /** * Fetch existing cgroups for the user from remote and save them to DB. */ diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index ffe631da97..4dabad3907 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -1,11 +1,15 @@ -import { assertionFailed } from "@/base/assert"; import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import type { EnteFile } from "../../types/file"; -import type { AnnotatedCGroup } from "./cgroups"; -import { faceDirection, type Face, type FaceIndex } from "./face"; +import type { Person } from "./cgroups"; +import { + faceDirection, + fileIDFromFaceID, + type Face, + type FaceIndex, +} from "./face"; import { dotProduct } from "./math"; /** @@ -236,21 +240,6 @@ const isSidewaysFace = (face: Face) => /** Generate a new cluster ID. */ const newClusterID = () => newNonSecureID("cluster_"); -/** - * Extract the fileID of the {@link EnteFile} to which the face belongs from its - * faceID. - * - * TODO-Cluster - duplicated with ml/index.ts - */ -const fileIDFromFaceID = (faceID: string) => { - const fileID = parseInt(faceID.split("_")[0] ?? ""); - if (isNaN(fileID)) { - assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); - return undefined; - } - return fileID; -}; - interface ClusteringState { clusterIDForFaceID: Map; clusterIndexForFaceID: Map; @@ -344,56 +333,35 @@ const toPeople = ( clusters: FaceCluster[], localFileByID: Map, faceForFaceID: Map, -) => { - // TODO-Cluster - Currently we're not syncing with remote or saving anything - // locally, so cgroups will be empty. Create a temporary (unsaved, unsynced) - // cgroup, one per cluster. +): Person[] => + clusters + .map((cluster) => { + const faces = cluster.faces.map((id) => + ensure(faceForFaceID.get(id)), + ); - const cgroups: AnnotatedCGroup[] = []; - for (const cluster of clusters) { - const faces = cluster.faces.map((id) => ensure(faceForFaceID.get(id))); - const topFace = faces.reduce((top, face) => - top.score > face.score ? top : face, - ); - cgroups.push({ - id: cluster.id, - name: undefined, - assigned: [cluster], - isHidden: false, - avatarFaceID: undefined, - displayFaceID: topFace.faceID, - }); - } + const faceIDs = cluster.faces; + const fileIDs = faceIDs.map((faceID) => + ensure(fileIDFromFaceID(faceID)), + ); - const clusterByID = new Map(clusters.map((c) => [c.id, c])); + const topFace = faces.reduce((top, face) => + top.score > face.score ? top : face, + ); - const people = cgroups - // TODO-Cluster - .map((cgroup) => ({ ...cgroup, name: cgroup.id })) - .map((cgroup) => { - if (!cgroup.name) return undefined; - const faceID = ensure(cgroup.displayFaceID); - const fileID = ensure(fileIDFromFaceID(faceID)); - const file = ensure(localFileByID.get(fileID)); - - const faceIDs = cgroup.assigned - .map(({ id }) => ensure(clusterByID.get(id))) - .flatMap((cluster) => cluster.faces); - const fileIDs = faceIDs - .map((faceID) => fileIDFromFaceID(faceID)) - .filter((fileID) => fileID !== undefined); + const displayFaceID = topFace.faceID; + const displayFaceFileID = ensure(fileIDFromFaceID(displayFaceID)); + const displayFaceFile = ensure( + localFileByID.get(displayFaceFileID), + ); return { - id: cgroup.id, - name: cgroup.name, + id: cluster.id, + name: undefined, faceIDs, fileIDs: [...new Set(fileIDs)], - displayFaceID: faceID, - displayFaceFile: file, + displayFaceID, + displayFaceFile, }; }) - .filter((c) => !!c) .sort((a, b) => b.faceIDs.length - a.faceIDs.length); - - return people; -}; diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.ts index 80b11853c8..0de4c6c6b4 100644 --- a/web/packages/new/photos/services/ml/face.ts +++ b/web/packages/new/photos/services/ml/face.ts @@ -7,6 +7,7 @@ // /* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { assertionFailed } from "@/base/assert"; import type { ElectronMLWorker } from "@/base/types/ipc"; import type { EnteFile } from "@/new/photos/types/file"; import { Matrix } from "ml-matrix"; @@ -149,7 +150,7 @@ export interface Face { * Finally, this face ID is not completely opaque. It consists of underscore * separated components, the first of which is the ID of the * {@link EnteFile} to which this face belongs. Client code can rely on this - * structure and can parse it if needed. + * structure and can parse it if needed using {@link fileIDFromFaceID}. */ faceID: string; /** @@ -228,6 +229,19 @@ export interface Box { height: number; } +/** + * Extract the fileID of the {@link EnteFile} to which the face belongs from its + * faceID. + */ +export const fileIDFromFaceID = (faceID: string) => { + const fileID = parseInt(faceID.split("_")[0] ?? ""); + if (isNaN(fileID)) { + assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); + return undefined; + } + return fileID; +}; + /** * Index faces in the given file. * diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 1d84ae7500..df70bbe30d 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -3,7 +3,6 @@ */ import { isDesktop } from "@/base/app"; -import { assertionFailed } from "@/base/assert"; import { blobCache } from "@/base/blob-cache"; import { ensureElectron } from "@/base/electron"; import { isDevBuild } from "@/base/env"; @@ -598,20 +597,6 @@ export const unidentifiedFaceIDs = async ( return index?.faces.map((f) => f.faceID) ?? []; }; -/** - * Extract the fileID of the {@link EnteFile} to which the face belongs from its - * faceID. - */ -// TODO-Cluster -export const fileIDFromFaceID = (faceID: string) => { - const fileID = parseInt(faceID.split("_")[0] ?? ""); - if (isNaN(fileID)) { - assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); - return undefined; - } - return fileID; -}; - /** * Return the cached face crop for the given face, regenerating it if needed. * From 2722b50cc04b6ba1de095ef99c679844f06aad4d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 17:17:10 +0530 Subject: [PATCH 15/16] Sequence --- web/apps/photos/src/pages/gallery.tsx | 3 -- web/apps/photos/src/services/sync.ts | 8 ++- .../new/photos/services/ml/cgroups.ts | 2 +- web/packages/new/photos/services/ml/index.ts | 54 +++++++++++++------ 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/web/apps/photos/src/pages/gallery.tsx b/web/apps/photos/src/pages/gallery.tsx index 216789f9b9..9a16723da2 100644 --- a/web/apps/photos/src/pages/gallery.tsx +++ b/web/apps/photos/src/pages/gallery.tsx @@ -14,7 +14,6 @@ import { getLocalFiles, getLocalTrashedFiles, } from "@/new/photos/services/files"; -import { wipClusterLocalOnce } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/cgroups"; import { filterSearchableFiles, @@ -681,8 +680,6 @@ export default function Gallery() { }; }, [selectAll, clearSelection]); - useEffect(() => wipClusterLocalOnce(), []); - const fileToCollectionsMap = useMemoSingleThreaded(() => { return constructFileToCollectionMap(files); }, [files]); diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 82a0f59d1d..012af2fe11 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -30,5 +30,9 @@ export const preFileInfoSync = async () => { * libraries after initial login), and the `preFileInfoSync`, which is called * before doing the file sync and thus should run immediately after login. */ -export const sync = () => - Promise.all([syncMapEnabled(), mlSync(), searchDataSync()]); +export const sync = async () => { + await Promise.all([syncMapEnabled(), searchDataSync()]); + // ML sync might take a very long time for initial indexing, so don't wait + // for it to finish. + void mlSync(); +}; diff --git a/web/packages/new/photos/services/ml/cgroups.ts b/web/packages/new/photos/services/ml/cgroups.ts index e7316e11f6..5966b9ca4c 100644 --- a/web/packages/new/photos/services/ml/cgroups.ts +++ b/web/packages/new/photos/services/ml/cgroups.ts @@ -144,7 +144,7 @@ export const syncCGroups = async () => { * @return A list of {@link Person}s, sorted by the number of files that they * reference. */ -export const updatedPeople = async () => { +export const peopleFromCGroups = async () => { if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return []; if (!(await wipClusterEnable())) return []; diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index df70bbe30d..1bed3f27ce 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -18,7 +18,7 @@ import { isInternalUser } from "../feature-flags"; import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import { setSearchPeople } from "../search"; import type { UploadItem } from "../upload/types"; -import { syncCGroups, updatedPeople, type Person } from "./cgroups"; +import { peopleFromCGroups, syncCGroups, type Person } from "./cgroups"; import { regenerateFaceCrops } from "./crop"; import { clearMLDB, getFaceIndex, getIndexableAndIndexedCounts } from "./db"; import { MLWorker } from "./worker"; @@ -51,6 +51,11 @@ class MLState { */ comlinkWorker: Promise> | undefined; + /** + * `true` if a sync is currently in progress. + */ + isSyncing = false; + /** * Subscriptions to {@link MLStatus} updates. * @@ -227,6 +232,7 @@ export const disableML = async () => { await updateIsMLEnabledRemote(false); setIsMLEnabledLocal(false); _state.isMLEnabled = false; + _state.isSyncing = false; await terminateMLWorker(); triggerStatusUpdate(); }; @@ -308,11 +314,36 @@ export const mlStatusSync = async () => { * least once prior to calling this in the sync sequence. */ export const mlSync = async () => { - if (_state.isMLEnabled) { - await Promise.all([worker().then((w) => w.sync()), syncCGroups()]).then( - updatePeople, - ); - } + if (!_state.isMLEnabled) return; + if (_state.isSyncing) return; + _state.isSyncing = true; + + // Dependency order for the sync + // + // files -> faces -> cgroups -> clusters + // + + // Fetch indexes, or index locally if needed. + await worker().then((w) => w.sync()); + + // Fetch existing cgroups. + await syncCGroups(); + + // Generate local clusters + // TODO-Cluster + // Warning - this is heavily WIP + wipClusterLocalOnce(); + + // Update our in-memory snapshot of people. + const namedPeople = await peopleFromCGroups(); + _state.peopleRemote = namedPeople; + updatePeopleSnapshot(); + + // Notify the search subsystem of the update. Since the search only used + // named cgroups, we only give it the people we got from cgroups. + setSearchPeople(namedPeople); + + _state.isSyncing = false; }; /** @@ -559,17 +590,6 @@ const setPeopleSnapshot = (snapshot: Person[] | undefined) => { _state.peopleListeners.forEach((l) => l()); }; -/** - * Update our in-memory snapshot of people, also notifying the search subsystem - * of the update. - */ -const updatePeople = async () => { - const people = await updatedPeople(); - _state.peopleRemote = people; - updatePeopleSnapshot(); - setSearchPeople(people); -}; - /** * Use CLIP to perform a natural language search over image embeddings. * From ed907c71f86c6e2b165c586628179fa3840b854b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 20 Sep 2024 17:45:49 +0530 Subject: [PATCH 16/16] Let tsc know --- web/packages/new/photos/services/ml/cgroups.ts | 8 ++++++-- web/packages/new/photos/services/ml/index.ts | 4 ++-- web/packages/new/photos/services/search/index.ts | 6 +++--- web/packages/new/photos/services/search/worker.ts | 15 +++++++++------ 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/web/packages/new/photos/services/ml/cgroups.ts b/web/packages/new/photos/services/ml/cgroups.ts index 5966b9ca4c..f33e27e53c 100644 --- a/web/packages/new/photos/services/ml/cgroups.ts +++ b/web/packages/new/photos/services/ml/cgroups.ts @@ -134,8 +134,12 @@ export const syncCGroups = async () => { await pullCGroups(masterKey); }; +export type NamedPerson = Omit & { + name: string; +}; + /** - * Construct in-memory "people" from the cgroups present locally. + * Construct in-memory {@link NamedPerson}s from the cgroups present locally. * * This function is meant to run after files, cgroups and faces have been synced * with remote. It then uses all the information in the local DBs to construct @@ -144,7 +148,7 @@ export const syncCGroups = async () => { * @return A list of {@link Person}s, sorted by the number of files that they * reference. */ -export const peopleFromCGroups = async () => { +export const namedPeopleFromCGroups = async (): Promise => { if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return []; if (!(await wipClusterEnable())) return []; diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 1bed3f27ce..03dc4a83ef 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -18,7 +18,7 @@ import { isInternalUser } from "../feature-flags"; import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import { setSearchPeople } from "../search"; import type { UploadItem } from "../upload/types"; -import { peopleFromCGroups, syncCGroups, type Person } from "./cgroups"; +import { namedPeopleFromCGroups, syncCGroups, type Person } from "./cgroups"; import { regenerateFaceCrops } from "./crop"; import { clearMLDB, getFaceIndex, getIndexableAndIndexedCounts } from "./db"; import { MLWorker } from "./worker"; @@ -335,7 +335,7 @@ export const mlSync = async () => { wipClusterLocalOnce(); // Update our in-memory snapshot of people. - const namedPeople = await peopleFromCGroups(); + const namedPeople = await namedPeopleFromCGroups(); _state.peopleRemote = namedPeople; updatePeopleSnapshot(); diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index 7d59b16971..759111292c 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -4,7 +4,7 @@ import { ComlinkWorker } from "@/base/worker/comlink-worker"; import { FileType } from "@/media/file-type"; import i18n, { t } from "i18next"; import { clipMatches, isMLEnabled, isMLSupported } from "../ml"; -import type { Person } from "../ml/cgroups"; +import type { NamedPerson } from "../ml/cgroups"; import type { LabelledFileType, LabelledSearchDateComponents, @@ -58,9 +58,9 @@ export const setSearchCollectionsAndFiles = (cf: SearchCollectionsAndFiles) => void worker().then((w) => w.setCollectionsAndFiles(cf)); /** - * Set the people that we should search across. + * Set the (named) people that we should search across. */ -export const setSearchPeople = (people: Person[]) => +export const setSearchPeople = (people: NamedPerson[]) => void worker().then((w) => w.setPeople(people)); /** diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 4fcbc4cb61..2150d9c228 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -2,7 +2,7 @@ import { HTTPError } from "@/base/http"; import type { Location } from "@/base/types"; import type { Collection } from "@/media/collection"; import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata"; -import type { Person } from "@/new/photos/services/ml/cgroups"; +import type { NamedPerson } from "@/new/photos/services/ml/cgroups"; import type { EnteFile } from "@/new/photos/types/file"; import { ensure } from "@/utils/ensure"; import { nullToUndefined } from "@/utils/transform"; @@ -37,7 +37,7 @@ export class SearchWorker { collections: [], files: [], }; - private people: Person[] = []; + private people: NamedPerson[] = []; /** * Fetch any state we might need when the actual search happens. @@ -62,9 +62,9 @@ export class SearchWorker { } /** - * Set the people that we should search across. + * Set the (named) people that we should search across. */ - setPeople(people: Person[]) { + setPeople(people: NamedPerson[]) { this.people = people; } @@ -122,7 +122,7 @@ const suggestionsForString = ( re: RegExp, searchString: string, { collections, files }: SearchCollectionsAndFiles, - people: Person[], + people: NamedPerson[], { locale, holidays, labelledFileTypes }: LocalizedSearchData, locationTags: LocationTag[], cities: City[], @@ -196,7 +196,10 @@ const fileCaptionSuggestion = ( : []; }; -const peopleSuggestions = (re: RegExp, people: Person[]): SearchSuggestion[] => +const peopleSuggestions = ( + re: RegExp, + people: NamedPerson[], +): SearchSuggestion[] => people .filter((p) => re.test(p.name)) .map((person) => ({ type: "person", person, label: person.name }));