[desktop] People suggestions - WIP - Part x/x (#3712)

This commit is contained in:
Manav Rathi
2024-10-15 17:15:28 +05:30
committed by GitHub
7 changed files with 286 additions and 140 deletions

View File

@@ -211,12 +211,12 @@ export const SuggestionFaceList: React.FC<SuggestionFaceListProps> = ({
return (
<SuggestionFaceList_>
{faces.map(({ file, faceID }) => (
<UnclusteredFace key={faceID}>
<SuggestionFace key={faceID}>
<FaceCropImageView
placeholderDimension={112}
{...{ file, faceID }}
/>
</UnclusteredFace>
</SuggestionFace>
))}
</SuggestionFaceList_>
);
@@ -225,7 +225,18 @@ export const SuggestionFaceList: React.FC<SuggestionFaceListProps> = ({
const SuggestionFaceList_ = styled("div")`
display: flex;
flex-wrap: wrap;
gap: 5px;
gap: 6px;
`;
const SuggestionFace = styled("div")`
width: 87px;
height: 87px;
border-radius: 50%;
overflow: hidden;
& > img {
width: 100%;
height: 100%;
}
`;
type FaceCropImageViewProps = PreviewableFace & {

View File

@@ -15,13 +15,14 @@ import log from "@/base/log";
import {
deleteCGroup,
renameCGroup,
suggestionsForPerson,
suggestionsAndChoicesForPerson,
} from "@/new/photos/services/ml";
import {
type CGroupPerson,
type ClusterPerson,
type Person,
type PersonSuggestion,
type PersonSuggestionsAndChoices,
type PreviewableCluster,
} from "@/new/photos/services/ml/people";
import { wait } from "@/utils/promise";
import OverflowMenu from "@ente/shared/components/OverflowMenu/menu";
@@ -30,8 +31,9 @@ import AddIcon from "@mui/icons-material/Add";
import CheckIcon from "@mui/icons-material/Check";
import ClearIcon from "@mui/icons-material/Clear";
import EditIcon from "@mui/icons-material/Edit";
import ListAltOutlined from "@mui/icons-material/ListAltOutlined";
import MoreHoriz from "@mui/icons-material/MoreHoriz";
import ListAltOutlinedIcon from "@mui/icons-material/ListAltOutlined";
import MoreHorizIcon from "@mui/icons-material/MoreHoriz";
import RestoreIcon from "@mui/icons-material/Restore";
import {
Dialog,
DialogActions,
@@ -47,10 +49,12 @@ import {
Typography,
} from "@mui/material";
import { t } from "i18next";
import React, { useEffect, useReducer } from "react";
import React, { useEffect, useReducer, useState } from "react";
import { isInternalUser } from "../../services/feature-flags";
import { useAppContext } from "../../types/context";
import { AddPersonDialog } from "../AddPersonDialog";
import { SpaceBetweenFlex } from "../mui";
import { SuggestionFaceList } from "../PeopleList";
import { SingleInputDialog } from "../SingleInputForm";
import type { GalleryBarImplProps } from "./BarImpl";
import { GalleryItemsHeaderAdapter, GalleryItemsSummary } from "./ListHeader";
@@ -115,12 +119,17 @@ const CGroupPersonHeader: React.FC<CGroupPersonHeaderProps> = ({
const cgroup = person.cgroup;
const { showMiniDialog } = useAppContext();
const [showReviewOption, setShowReviewOption] = useState(false);
const { show: showNameInput, props: nameInputVisibilityProps } =
useModalVisibility();
const { show: showSuggestions, props: suggestionsVisibilityProps } =
useModalVisibility();
useEffect(() => {
void isInternalUser().then((b) => setShowReviewOption(b));
}, []);
const handleRename = (name: string) => renameCGroup(cgroup, name);
const handleReset = () =>
@@ -152,7 +161,7 @@ const CGroupPersonHeader: React.FC<CGroupPersonHeaderProps> = ({
/>
<OverflowMenu
ariaControls={"person-options"}
triggerButtonIcon={<MoreHoriz />}
triggerButtonIcon={<MoreHorizIcon />}
>
<OverflowMenuOption
startIcon={<EditIcon />}
@@ -168,9 +177,9 @@ const CGroupPersonHeader: React.FC<CGroupPersonHeaderProps> = ({
>
{pt("Reset")}
</OverflowMenuOption>
{process.env.NEXT_PUBLIC_ENTE_WIP_CL /* TODO-Cluster */ && (
{showReviewOption /* TODO-Cluster */ && (
<OverflowMenuOption
startIcon={<ListAltOutlined />}
startIcon={<ListAltOutlinedIcon />}
centerAlign
onClick={showSuggestions}
>
@@ -232,7 +241,7 @@ const ClusterPersonHeader: React.FC<ClusterPersonHeaderProps> = ({
<OverflowMenu
ariaControls={"person-options"}
triggerButtonIcon={<MoreHoriz />}
triggerButtonIcon={<MoreHorizIcon />}
>
<OverflowMenuOption
startIcon={<AddIcon />}
@@ -268,33 +277,45 @@ interface SuggestionsDialogState {
*/
fetchFailed: boolean;
/**
* List of clusters (suitably augmented for the UI display) which might
* belong to the person, and being offered to the user as suggestions.
* True if we should show the previously saved choice view instead of the
* new suggestions.
*/
suggestions: PersonSuggestion[];
showChoices: boolean;
/** Fetched choices. */
choices: SCItem[];
/** Fetched suggestions. */
suggestions: SCItem[];
/**
* An entry corresponding to each clusters (suggestions) that the user has
* either explicitly accepted or rejected.
* An entry corresponding to each
* - saved choice for which the user has changed their mind.
* - suggestion that the user has either explicitly accepted or rejected.
*/
markedSuggestionIDs: Map<string, NonNullable<SuggestionMark>>;
marks: Map<string, boolean | undefined>;
}
type SuggestionMark = "yes" | "no" | undefined;
type SCItem = PreviewableCluster & { fixed?: boolean; accepted?: boolean };
type SuggestionsDialogAction =
| { type: "fetch"; personID: string }
| { type: "fetchFailed"; personID: string }
| { type: "fetched"; personID: string; suggestions: PersonSuggestion[] }
| { type: "mark"; suggestion: PersonSuggestion; value: SuggestionMark }
| {
type: "fetched";
personID: string;
suggestionsAndChoices: PersonSuggestionsAndChoices;
}
| { type: "mark"; item: SCItem; value: boolean | undefined }
| { type: "save" }
| { type: "toggleHistory" }
| { type: "close" };
const initialSuggestionsDialogState: SuggestionsDialogState = {
activity: undefined,
personID: undefined,
fetchFailed: false,
showChoices: false,
choices: [],
suggestions: [],
markedSuggestionIDs: new Map(),
marks: new Map(),
};
const suggestionsDialogReducer = (
@@ -304,11 +325,12 @@ const suggestionsDialogReducer = (
switch (action.type) {
case "fetch":
return {
...initialSuggestionsDialogState,
choices: [],
suggestions: [],
marks: new Map(),
activity: "fetching",
personID: action.personID,
fetchFailed: false,
suggestions: [],
markedSuggestionIDs: new Map(),
};
case "fetchFailed":
if (action.personID != state.personID) return state;
@@ -318,21 +340,27 @@ const suggestionsDialogReducer = (
return {
...state,
activity: undefined,
suggestions: action.suggestions,
choices: action.suggestionsAndChoices.choices,
suggestions: action.suggestionsAndChoices.suggestions,
};
case "mark": {
const markedSuggestionIDs = new Map(state.markedSuggestionIDs);
const id = action.suggestion.id;
if (action.value == "yes" || action.value == "no") {
markedSuggestionIDs.set(id, action.value);
const marks = new Map(state.marks);
const { item, value } = action;
if (item.accepted === undefined && value === undefined) {
// If this was a suggestion, prune marks created as a result of
// the user toggling the item back to its original unset state.
marks.delete(item.id);
} else if (item.accepted && value === item.accepted) {
// If this is a choice, prune marks which match the choice's
// accepted state.
marks.delete(item.id);
} else {
markedSuggestionIDs.delete(id);
marks.set(item.id, value);
}
return {
...state,
markedSuggestionIDs,
};
return { ...state, marks };
}
case "toggleHistory":
return { ...state, showChoices: !state.showChoices };
case "save":
return { ...state, activity: "saving" };
case "close":
@@ -360,7 +388,7 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
const isSmallWidth = useIsSmallWidth();
const hasUnsavedChanges = state.markedSuggestionIDs.size > 0;
const hasUnsavedChanges = state.marks.size > 0;
const resetPersonAndClose = () => {
dispatch({ type: "close" });
@@ -380,10 +408,11 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
const go = async () => {
try {
const suggestions = await suggestionsForPerson(person);
dispatch({ type: "fetched", personID, suggestions });
const suggestionsAndChoices =
await suggestionsAndChoicesForPerson(person);
dispatch({ type: "fetched", personID, suggestionsAndChoices });
} catch (e) {
log.error("Failed to generate suggestions", e);
log.error("Failed to fetch suggestions and choices", e);
dispatch({ type: "fetchFailed", personID });
}
};
@@ -410,8 +439,8 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
resetPersonAndClose();
};
const handleMark = (suggestion: PersonSuggestion, value: SuggestionMark) =>
dispatch({ type: "mark", suggestion, value });
const handleMark = (item: SCItem, value: boolean | undefined) =>
dispatch({ type: "mark", item, value });
const handleSave = async () => {
try {
@@ -434,10 +463,48 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
fullScreen={isSmallWidth}
PaperProps={{ sx: { minHeight: "80svh" } }}
>
<DialogTitle sx={{ "&&&": { py: "20px" } }}>
{person.name && pt(`${person.name}?`)}
</DialogTitle>
<DialogContent dividers sx={{ display: "flex" }}>
<SpaceBetweenFlex
sx={{
padding: "20px 16px 16px 16px",
backgroundColor: state.showChoices
? (theme) => theme.colors.fill.faint
: "transparent",
}}
>
<Stack sx={{ gap: "8px" }}>
<DialogTitle sx={{ "&&&": { p: 0 } }}>
{state.showChoices
? pt("Saved choices")
: pt("Review suggestions")}
</DialogTitle>
<Typography color="text.muted">
{person.name ?? " "}
</Typography>
</Stack>
{state.choices.length > 1 && (
<IconButton
disableTouchRipple
onClick={() => dispatch({ type: "toggleHistory" })}
aria-label={
!state.showChoices
? pt("Saved suggestions")
: pt("Review suggestions")
}
sx={{
backgroundColor: state.showChoices
? (theme) => theme.colors.fill.muted
: "transparent",
}}
>
<RestoreIcon />
</IconButton>
)}
</SpaceBetweenFlex>
<DialogContent
/* Reset scroll position on switching view */
key={`${state.showChoices}`}
sx={{ display: "flex", "&&&": { pt: 0 } }}
>
{state.activity == "fetching" ? (
<CenteredBox>
<ActivityIndicator>
@@ -448,6 +515,12 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
<CenteredBox>
<ErrorIndicator />
</CenteredBox>
) : state.showChoices ? (
<SuggestionOrChoiceList
items={state.choices}
marks={state.marks}
onMarkItem={handleMark}
/>
) : state.suggestions.length == 0 ? (
<CenteredBox>
<Typography
@@ -458,10 +531,10 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
</Typography>
</CenteredBox>
) : (
<SuggestionsList
suggestions={state.suggestions}
markedSuggestionIDs={state.markedSuggestionIDs}
onMarkSuggestion={handleMark}
<SuggestionOrChoiceList
items={state.suggestions}
marks={state.marks}
onMarkItem={handleMark}
/>
)}
</DialogContent>
@@ -480,61 +553,74 @@ const SuggestionsDialog: React.FC<SuggestionsDialogProps> = ({
color={"accent"}
onClick={handleSave}
>
{t("save")}
{hasUnsavedChanges ? pt("TODO Not impl") : t("save")}
</LoadingButton>
</DialogActions>
</Dialog>
);
};
type SuggestionsListProps = Pick<
SuggestionsDialogState,
"suggestions" | "markedSuggestionIDs"
> & {
interface SuggestionOrChoiceListProps {
items: SCItem[];
marks: Map<string, boolean | undefined>;
/**
* Callback invoked when the user toggles the value associated with the
* given suggestion.
* Callback invoked when the user changes the value associated with the
* given suggestion or choice.
*/
onMarkSuggestion: (
suggestion: PersonSuggestion,
value: SuggestionMark,
) => void;
};
onMarkItem: (item: SCItem, value: boolean | undefined) => void;
}
const SuggestionsList: React.FC<SuggestionsListProps> = ({
suggestions,
markedSuggestionIDs,
onMarkSuggestion,
const SuggestionOrChoiceList: React.FC<SuggestionOrChoiceListProps> = ({
items,
marks,
onMarkItem,
}) => (
<List sx={{ width: "100%" }}>
{suggestions.map((suggestion) => (
<List dense sx={{ width: "100%" }}>
{items.map((item) => (
<ListItem
key={item.id}
sx={{
paddingInline: 0,
paddingBlockEnd: "24px",
justifyContent: "space-between",
}}
key={suggestion.id}
>
<Typography>{`${suggestion.previewFaces.length} faces ntaoheu naoehtu aosnehu asoenuh aoenuht`}</Typography>
<ToggleButtonGroup
value={markedSuggestionIDs.get(suggestion.id)}
exclusive
onChange={(_, v) =>
onMarkSuggestion(
suggestion,
// Dance for TypeScript to recognize the type.
v == "yes" ? "yes" : v == "no" ? "no" : undefined,
)
}
>
<ToggleButton value="yes" aria-label={pt("Yes")}>
<CheckIcon />
</ToggleButton>
<ToggleButton value="no" aria-label={t("no")}>
<ClearIcon />
</ToggleButton>
</ToggleButtonGroup>
<Stack sx={{ gap: "10px" }}>
<Typography variant="small" color="text.muted">
{/* Use the face count as as stand-in for the photo count */}
{t("photos_count", { count: item.faces.length })}
</Typography>
<SuggestionFaceList faces={item.previewFaces} />
</Stack>
{!item.fixed && (
<ToggleButtonGroup
value={fromItemValue(item, marks)}
exclusive
onChange={(_, v) => onMarkItem(item, toItemValue(v))}
>
<ToggleButton value="no" aria-label={t("no")}>
<ClearIcon />
</ToggleButton>
<ToggleButton value="yes" aria-label={pt("Yes")}>
<CheckIcon />
</ToggleButton>
</ToggleButtonGroup>
)}
</ListItem>
))}
</List>
);
const fromItemValue = (
item: SCItem,
marks: Map<string, boolean | undefined>,
) => {
// Use the in-memory state if available. For choices, fallback to their
// original state.
const resolved = marks.has(item.id) ? marks.get(item.id) : item.accepted;
return resolved ? "yes" : resolved === false ? "no" : undefined;
};
const toItemValue = (v: unknown) =>
// This dance is needed for TypeScript to recognize the type.
v == "yes" ? true : v == "no" ? false : undefined;

View File

@@ -133,7 +133,7 @@ const normalized = (embedding: Float32Array) => {
* The result can also be `undefined`, which indicates that the download for the
* ML model is still in progress (trying again later should succeed).
*/
export const clipMatches = async (
export const _clipMatches = async (
searchPhrase: string,
electron: ElectronMLWorker,
): Promise<CLIPMatches | undefined> => {

View File

@@ -62,7 +62,7 @@ export type ClusterFace = Omit<Face, "embedding"> & {
* other interactions with the worker (where this code runs) do not get stalled
* while clustering is in progress.
*/
export const clusterFaces = async (
export const _clusterFaces = async (
faceIndexes: FaceIndex[],
localFiles: EnteFile[],
onProgress: (progress: ClusteringProgress) => void,

View File

@@ -794,5 +794,5 @@ export const deleteCGroup = async ({ id }: CGroup) => {
*
* The suggestion computation happens in a web worker.
*/
export const suggestionsForPerson = async (person: CGroupPerson) =>
worker().then((w) => w.suggestionsForPerson(person));
export const suggestionsAndChoicesForPerson = async (person: CGroupPerson) =>
worker().then((w) => w.suggestionsAndChoicesForPerson(person));

View File

@@ -1,9 +1,11 @@
import { assertionFailed } from "@/base/assert";
import log from "@/base/log";
import type { EnteFile } from "@/media/file";
import { shuffled } from "@/utils/array";
import { ensure } from "@/utils/ensure";
import { getLocalFiles } from "../files";
import { savedCGroups, type CGroup } from "../user-entity";
import type { FaceCluster } from "./cluster";
import { type FaceCluster } from "./cluster";
import { savedFaceClusters, savedFaceIndexes } from "./db";
import { fileIDFromFaceID } from "./face";
import { dotProduct } from "./math";
@@ -322,28 +324,45 @@ export const filterNamedPeople = (people: Person[]): NamedPerson[] => {
return namedPeople;
};
export interface PersonSuggestion {
export type PreviewableCluster = FaceCluster & {
/**
* The ID of the suggestion. This is the same as the cluster's ID,
* duplicated here for the UI's convenience.
*/
id: string;
/**
* The underlying {@link FaceCluster} that is being offered as the
* suggestion.
*/
cluster: FaceCluster;
/**
* A list of up to 3 "preview" faces for this cluster, each annotated with
* A list of up to 3 "preview" faces for the cluster, each annotated with
* the corresponding {@link EnteFile} that contains them.
*/
previewFaces: PreviewableFace[];
};
export interface PersonSuggestionsAndChoices {
/**
* Previously saved choices.
*
* These are clusters (sorted by size) that the user had previously merged
* or explicitly ignored from the person under consideration.
*
* The {@link accepted} flag will true for the entries that correspond to
* accepted clusters, and false for those that the user had ignored.
*
* This array is guaranteed to be non-empty, and it is guaranteed that the
* first item is a merged cluster (i.e. a cluster for which accepted is
* true), even if there exists an ignored cluster with a larger size. The
* rest of the entries are intermixed and sorted by size normally.
*
* For convenience of the UI, teh first entry will have also have the
* {@link fixed} flag set.
*/
choices: (PreviewableCluster & { fixed?: boolean; accepted: boolean })[];
/**
* New suggestions to offer to the user.
*/
suggestions: PreviewableCluster[];
}
/**
* Returns suggestions for the given person.
* Returns suggestions and existing choices for the given person.
*/
export const suggestionsForPerson = async (person: CGroupPerson) => {
export const _suggestionsAndChoicesForPerson = async (
person: CGroupPerson,
): Promise<PersonSuggestionsAndChoices> => {
const startTime = Date.now();
const personClusters = person.cgroup.data.assigned;
@@ -371,6 +390,12 @@ export const suggestionsForPerson = async (person: CGroupPerson) => {
.flat()
.filter((e) => !!e);
// Randomly sample faces to limit the O(n^2) cost.
const sampledPersonFaceEmbeddings = shuffled(personFaceEmbeddings).slice(
0,
100,
);
const suggestedClusters: FaceCluster[] = [];
for (const cluster of clusters) {
const { id, faces } = cluster;
@@ -384,7 +409,7 @@ export const suggestionsForPerson = async (person: CGroupPerson) => {
for (const fi of faces) {
const ei = embeddingByFaceID.get(fi);
if (!ei) continue;
for (const ej of personFaceEmbeddings) {
for (const ej of sampledPersonFaceEmbeddings) {
const csim = dotProduct(ei, ej);
if (csim >= 0.6) {
suggest = true;
@@ -397,41 +422,65 @@ export const suggestionsForPerson = async (person: CGroupPerson) => {
if (suggest) suggestedClusters.push(cluster);
}
suggestedClusters.sort((a, b) => b.faces.length - a.faces.length);
// Annotate the clusters with the information that the UI needs to show its
// preview faces.
const files = await getLocalFiles("normal");
const fileByID = new Map(files.map((f) => [f.id, f]));
const suggestions = suggestedClusters.map((cluster) => {
const previewFaces = cluster.faces
.slice(0, 3)
.map((faceID) => {
const fileID = fileIDFromFaceID(faceID);
if (!fileID) {
assertionFailed();
return undefined;
}
const file = fileByID.get(fileID);
if (!file) {
// TODO-Cluster: This might be a hidden/trash file, so this
// assert is not appropriate, we instead need a "until 3".
// assertionFailed();
return undefined;
}
return { file, faceID };
})
.filter((f) => !!f);
const toPreviewable = (cluster: FaceCluster) => {
const previewFaces: PreviewableFace[] = [];
for (const faceID of cluster.faces) {
const fileID = fileIDFromFaceID(faceID);
if (!fileID) {
assertionFailed();
continue;
}
const id = cluster.id;
return { id, cluster, previewFaces };
});
const file = fileByID.get(fileID);
if (!file) {
// This might be a hidden/trash file, and it is thus not
// appropriate to use it as a preview file anyway.
continue;
}
previewFaces.push({ file, faceID });
if (previewFaces.length == 4) break;
}
return { ...cluster, previewFaces };
};
const sortBySize = (entries: { faces: unknown[] }[]) =>
entries.sort((a, b) => b.faces.length - a.faces.length);
const acceptedChoices = personClusters
.map(toPreviewable)
.map((p) => ({ ...p, accepted: true }));
sortBySize(acceptedChoices);
const ignoredChoices = ignoredClusters
.map(toPreviewable)
.map((p) => ({ ...p, accepted: false }));
// Ensure that the first item in the choices is not an ignored one, even if
// that is what we'd have ended up with if we sorted by size.
const firstChoice = { ...ensure(acceptedChoices[0]), fixed: true };
const restChoices = acceptedChoices.slice(1).concat(ignoredChoices);
sortBySize(restChoices);
const choices = [firstChoice, ...restChoices];
sortBySize(suggestedClusters);
// Limit to the number of suggestions shown in a single go.
const suggestions = suggestedClusters.slice(0, 80).map(toPreviewable);
log.info(
`Generated ${suggestions.length} suggestions for ${person.id} (${Date.now() - startTime} ms)`,
);
return suggestions;
return { choices, suggestions };
};

View File

@@ -18,14 +18,14 @@ import {
type ImageBitmapAndData,
} from "./blob";
import {
_clipMatches,
clearCachedCLIPIndexes,
clipIndexingVersion,
clipMatches,
indexCLIP,
type CLIPIndex,
} from "./clip";
import {
clusterFaces,
_clusterFaces,
reconcileClusters,
type ClusteringProgress,
} from "./cluster";
@@ -44,7 +44,7 @@ import {
type RawRemoteMLData,
type RemoteMLData,
} from "./ml-data";
import { suggestionsForPerson, type CGroupPerson } from "./people";
import { _suggestionsAndChoicesForPerson, type CGroupPerson } from "./people";
import type { CLIPMatches, MLWorkerDelegate } from "./worker-types";
/**
@@ -207,7 +207,7 @@ export class MLWorker {
* Find {@link CLIPMatches} for a given normalized {@link searchPhrase}.
*/
async clipMatches(searchPhrase: string): Promise<CLIPMatches | undefined> {
return clipMatches(searchPhrase, ensure(this.electron));
return _clipMatches(searchPhrase, ensure(this.electron));
}
private async tick() {
@@ -326,7 +326,7 @@ export class MLWorker {
* cgroups if needed.
*/
async clusterFaces(masterKey: Uint8Array) {
const clusters = await clusterFaces(
const clusters = await _clusterFaces(
await savedFaceIndexes(),
await getAllLocalFiles(),
(progress) => this.updateClusteringProgress(progress),
@@ -341,10 +341,10 @@ export class MLWorker {
}
/**
* Return suggestions for the given cgroup {@link person}.
* Return suggestions and choices for the given cgroup {@link person}.
*/
async suggestionsForPerson(person: CGroupPerson) {
return suggestionsForPerson(person);
async suggestionsAndChoicesForPerson(person: CGroupPerson) {
return _suggestionsAndChoicesForPerson(person);
}
}