From 90494c6144c65ae520ced4c5d137464e2e52c125 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 16 Oct 2024 10:50:01 +0530 Subject: [PATCH 1/4] Filter empty suggestions --- web/packages/new/photos/services/ml/people.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/web/packages/new/photos/services/ml/people.ts b/web/packages/new/photos/services/ml/people.ts index 16b36a0232..44d8a8b7f4 100644 --- a/web/packages/new/photos/services/ml/people.ts +++ b/web/packages/new/photos/services/ml/people.ts @@ -449,21 +449,28 @@ export const _suggestionsAndChoicesForPerson = async ( if (previewFaces.length == 4) break; } + if (previewFaces.length == 0) return undefined; + return { ...cluster, previewFaces }; }; + const toPreviewableList = (clusters: FaceCluster[]) => + clusters.map(toPreviewable).filter((p) => !!p); + const sortBySize = (entries: { faces: unknown[] }[]) => entries.sort((a, b) => b.faces.length - a.faces.length); - const acceptedChoices = personClusters - .map(toPreviewable) - .map((p) => ({ ...p, accepted: true })); + const acceptedChoices = toPreviewableList(personClusters).map((p) => ({ + ...p, + accepted: true, + })); sortBySize(acceptedChoices); - const ignoredChoices = ignoredClusters - .map(toPreviewable) - .map((p) => ({ ...p, accepted: false })); + const ignoredChoices = toPreviewableList(ignoredClusters).map((p) => ({ + ...p, + accepted: false, + })); // Ensure that the first item in the choices is not an ignored one, even if // that is what we'd have ended up with if we sorted by size. @@ -476,7 +483,7 @@ export const _suggestionsAndChoicesForPerson = async ( sortBySize(suggestedClusters); // Limit to the number of suggestions shown in a single go. - const suggestions = suggestedClusters.slice(0, 80).map(toPreviewable); + const suggestions = toPreviewableList(suggestedClusters.slice(0, 80)); log.info( `Generated ${suggestions.length} suggestions for ${person.id} (${Date.now() - startTime} ms)`, From e697b863e7bff80dc2c9f911c962d2afe7d00933 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 16 Oct 2024 10:57:12 +0530 Subject: [PATCH 2/4] Debugging timings --- web/packages/new/photos/services/ml/people.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/web/packages/new/photos/services/ml/people.ts b/web/packages/new/photos/services/ml/people.ts index 44d8a8b7f4..fb93300f2c 100644 --- a/web/packages/new/photos/services/ml/people.ts +++ b/web/packages/new/photos/services/ml/people.ts @@ -363,6 +363,7 @@ export interface PersonSuggestionsAndChoices { export const _suggestionsAndChoicesForPerson = async ( person: CGroupPerson, ): Promise => { + console.time("prep"); const startTime = Date.now(); const personClusters = person.cgroup.data.assigned; @@ -396,6 +397,10 @@ export const _suggestionsAndChoicesForPerson = async ( 100, ); + console.timeEnd("prep"); + + console.time("loop"); + const suggestedClusters: FaceCluster[] = []; for (const cluster of clusters) { const { id, faces } = cluster; @@ -422,6 +427,10 @@ export const _suggestionsAndChoicesForPerson = async ( if (suggest) suggestedClusters.push(cluster); } + console.timeEnd("loop"); + + console.time("post"); + // Annotate the clusters with the information that the UI needs to show its // preview faces. @@ -485,6 +494,8 @@ export const _suggestionsAndChoicesForPerson = async ( // Limit to the number of suggestions shown in a single go. const suggestions = toPreviewableList(suggestedClusters.slice(0, 80)); + console.timeEnd("post"); + log.info( `Generated ${suggestions.length} suggestions for ${person.id} (${Date.now() - startTime} ms)`, ); From 56365ac520210ed572eaeff96c20690361f718ff Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 16 Oct 2024 11:53:51 +0530 Subject: [PATCH 3/4] Use median check --- web/packages/new/photos/services/ml/people.ts | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/web/packages/new/photos/services/ml/people.ts b/web/packages/new/photos/services/ml/people.ts index fb93300f2c..748b6a9f51 100644 --- a/web/packages/new/photos/services/ml/people.ts +++ b/web/packages/new/photos/services/ml/people.ts @@ -392,16 +392,13 @@ export const _suggestionsAndChoicesForPerson = async ( .filter((e) => !!e); // Randomly sample faces to limit the O(n^2) cost. - const sampledPersonFaceEmbeddings = shuffled(personFaceEmbeddings).slice( - 0, - 100, - ); + const sampledPersonEmbeddings = randomSample(personFaceEmbeddings, 50); console.timeEnd("prep"); console.time("loop"); - const suggestedClusters: FaceCluster[] = []; + const candidateClustersAndSimilarity: [FaceCluster, number][] = []; for (const cluster of clusters) { const { id, faces } = cluster; @@ -410,27 +407,53 @@ export const _suggestionsAndChoicesForPerson = async ( if (personClusterIDs.has(id)) continue; if (ignoredClusterIDs.has(id)) continue; - let suggest = false; - for (const fi of faces) { - const ei = embeddingByFaceID.get(fi); - if (!ei) continue; - for (const ej of sampledPersonFaceEmbeddings) { - const csim = dotProduct(ei, ej); - if (csim >= 0.6) { - suggest = true; - break; + if (process.env.NEXT_PUBLIC_ENTE_WIP_CL_TODO) { + /* direct compare method TODO-Cluster remove me */ + let suggest = false; + for (const fi of faces) { + const ei = embeddingByFaceID.get(fi); + if (!ei) continue; + for (const ej of sampledPersonEmbeddings) { + const csim = dotProduct(ei, ej); + if (csim >= 0.6) { + suggest = true; + break; + } + } + if (suggest) break; + } + + if (suggest) candidateClustersAndSimilarity.push([cluster, 0]); + } else { + const sampledOtherEmbeddings = randomSample(faces, 50) + .map((id) => embeddingByFaceID.get(id)) + .filter((e) => !!e); + + /* cosine similarities */ + const csims: number[] = []; + for (const other of sampledOtherEmbeddings) { + for (const embedding of sampledPersonEmbeddings) { + csims.push(dotProduct(embedding, other)); } } - if (suggest) break; - } + csims.sort(); - if (suggest) suggestedClusters.push(cluster); + if (csims.length == 0) continue; + + const medianSim = ensure(csims[Math.floor(csims.length / 2)]); + if (medianSim > 0.48) { + candidateClustersAndSimilarity.push([cluster, medianSim]); + } + } } console.timeEnd("loop"); console.time("post"); + candidateClustersAndSimilarity.sort(([, a], [, b]) => b - a); + const suggestedClusters = candidateClustersAndSimilarity.map(([c]) => c); + // Annotate the clusters with the information that the UI needs to show its // preview faces. @@ -502,3 +525,6 @@ export const _suggestionsAndChoicesForPerson = async ( return { choices, suggestions }; }; + +const randomSample = (items: T[], n: number) => + items.length < n ? items : shuffled(items).slice(0, n); From 6578d4e5708f8f5a59c78814aac073c1d69ea9cc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 16 Oct 2024 12:05:52 +0530 Subject: [PATCH 4/4] Sort by median sim --- web/packages/new/photos/services/ml/people.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/ml/people.ts b/web/packages/new/photos/services/ml/people.ts index 748b6a9f51..80f8a5795c 100644 --- a/web/packages/new/photos/services/ml/people.ts +++ b/web/packages/new/photos/services/ml/people.ts @@ -513,7 +513,7 @@ export const _suggestionsAndChoicesForPerson = async ( const choices = [firstChoice, ...restChoices]; - sortBySize(suggestedClusters); + // sortBySize(suggestedClusters); // Limit to the number of suggestions shown in a single go. const suggestions = toPreviewableList(suggestedClusters.slice(0, 80));