TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'fix_issuer_name_encoding' into auth_trash

Browse Source
This commit is contained in:
Neeraj Gupta
2024-09-13 10:27:24 +05:30
parent 08a77a2def 75c3bc1c84
commit 0275d08e27
26 changed files with 712 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
auth/lib/models/code.dart
View File

@@ -83,6 +83,7 @@ class Code {
final Type updatedType = type ?? this.type;
final int updatedCounter = counter ?? this.counter;
final CodeDisplay updatedDisplay = display ?? this.display;
final String encodedIssuer = Uri.encodeQueryComponent(updateIssuer);
return Code(
updateAccount,
@@ -94,7 +95,7 @@ class Code {
updatedType,
updatedCounter,
"otpauth://${updatedType.name}/$updateIssuer:$updateAccount?algorithm=${updatedAlgo.name}"
"&digits=$updatedDigits&issuer=$updateIssuer"
"&digits=$updatedDigits&issuer=$encodedIssuer"
"&period=$updatePeriod&secret=$updatedSecret${updatedType == Type.hotp ? "&counter=$updatedCounter" : ""}",
generatedID: generatedID,
display: updatedDisplay,
@@ -109,6 +110,7 @@ class Code {
CodeDisplay? display,
int digits,
) {
final String encodedIssuer = Uri.encodeQueryComponent(issuer);
return Code(
account,
issuer,
@@ -118,7 +120,7 @@ class Code {
Algorithm.sha1,
type,
0,
"otpauth://${type.name}/$issuer:$account?algorithm=SHA1&digits=$digits&issuer=$issuer&period=30&secret=$secret",
"otpauth://${type.name}/$issuer:$account?algorithm=SHA1&digits=$digits&issuer=$encodedIssuer&period=30&secret=$secret",
display: display ?? CodeDisplay(),
);
}

29
auth/lib/onboarding/view/setup_enter_secret_key_page.dart
View File

@@ -38,7 +38,7 @@ class _SetupEnterSecretKeyPageState extends State<SetupEnterSecretKeyPage> {
late TextEditingController _secretController;
late TextEditingController _notesController;
late bool _secretKeyObscured;
late List<String> tags = [...?widget.code?.display.tags];
late List<String> selectedTags = [...?widget.code?.display.tags];
List<String> allTags = [];
StreamSubscription<CodesUpdatedEvent>? _streamSubscription;
@@ -272,13 +272,6 @@ class _SetupEnterSecretKeyPageState extends State<SetupEnterSecretKeyPage> {
),
],
),
const SizedBox(height: 24),
Text(
l10n.tags,
style: const TextStyle(
fontWeight: FontWeight.bold,
),
),
const SizedBox(height: 10),
Wrap(
spacing: 12,
@@ -288,14 +281,14 @@ class _SetupEnterSecretKeyPageState extends State<SetupEnterSecretKeyPage> {
(e) => TagChip(
label: e,
action: TagChipAction.check,
state: tags.contains(e)
state: selectedTags.contains(e)
? TagChipState.selected
: TagChipState.unselected,
onTap: () {
if (tags.contains(e)) {
tags.remove(e);
if (selectedTags.contains(e)) {
selectedTags.remove(e);
} else {
tags.add(e);
selectedTags.add(e);
}
setState(() {});
},
@@ -308,11 +301,12 @@ class _SetupEnterSecretKeyPageState extends State<SetupEnterSecretKeyPage> {
builder: (BuildContext context) {
return AddTagDialog(
onTap: (tag) {
if (allTags.contains(tag) && tags.contains(tag)) {
return;
final exist = allTags.contains(tag);
if (exist && selectedTags.contains(tag)) {
return Navigator.pop(context);
}
allTags.add(tag);
tags.add(tag);
if (!exist) allTags.add(tag);
selectedTags.add(tag);
setState(() {});
Navigator.pop(context);
},
@@ -366,7 +360,8 @@ class _SetupEnterSecretKeyPageState extends State<SetupEnterSecretKeyPage> {
final isStreamCode = issuer.toLowerCase() == "steam" ||
issuer.toLowerCase().contains('steampowered.com');
final CodeDisplay display =
widget.code?.display.copyWith(tags: tags) ?? CodeDisplay(tags: tags);
widget.code?.display.copyWith(tags: selectedTags) ??
CodeDisplay(tags: selectedTags);
display.note = notes;
if (widget.code != null && widget.code!.secret != secret) {
ButtonResult? result = await showChoiceActionSheet(

4
mobile/lib/ui/components/home_header_widget.dart
View File

@@ -86,8 +86,8 @@ class _HomeHeaderWidgetState extends State<HomeHeaderWidget> {
unawaited(
routeToPage(
context,
BackupFolderSelectionPage(
buttonText: S.of(context).backup,
const BackupFolderSelectionPage(
isFirstBackup: false,
),
),
);

36
mobile/lib/ui/home/loading_photos_widget.dart
View File

@@ -25,7 +25,7 @@ class LoadingPhotosWidget extends StatefulWidget {
class _LoadingPhotosWidgetState extends State<LoadingPhotosWidget> {
late StreamSubscription<SyncStatusUpdate> _firstImportEvent;
late StreamSubscription<LocalImportProgressEvent> _importProgressEvent;
StreamSubscription<LocalImportProgressEvent>? _importProgressEvent;
int _currentPage = 0;
late String _loadingMessage;
final PageController _pageController = PageController(
@@ -38,7 +38,6 @@ class _LoadingPhotosWidgetState extends State<LoadingPhotosWidget> {
@override
void initState() {
super.initState();
_loadingMessage = S.of(context).loadingYourPhotos;
Future.delayed(const Duration(seconds: 60), () {
oneMinuteOnScreen.value = true;
});
@@ -51,21 +50,15 @@ class _LoadingPhotosWidgetState extends State<LoadingPhotosWidget> {
// ignore: unawaited_futures
routeToPage(
context,
BackupFolderSelectionPage(
const BackupFolderSelectionPage(
isOnboarding: true,
buttonText: S.of(context).startBackup,
isFirstBackup: true,
),
);
}
}
});
_importProgressEvent =
Bus.instance.on<LocalImportProgressEvent>().listen((event) {
_loadingMessage = S.of(context).processingImport(event.folderName);
if (mounted) {
setState(() {});
}
});
_didYouKnowTimer =
Timer.periodic(const Duration(seconds: 5), (Timer timer) {
if (!mounted) {
@@ -85,10 +78,26 @@ class _LoadingPhotosWidgetState extends State<LoadingPhotosWidget> {
});
}
@override
void didChangeDependencies() {
super.didChangeDependencies();
if (_importProgressEvent != null) {
_importProgressEvent!.cancel();
} else {
_importProgressEvent =
Bus.instance.on<LocalImportProgressEvent>().listen((event) {
_loadingMessage = S.of(context).processingImport(event.folderName);
if (mounted) {
setState(() {});
}
});
}
}
@override
void dispose() {
_firstImportEvent.cancel();
_importProgressEvent.cancel();
_importProgressEvent?.cancel();
_didYouKnowTimer.cancel();
oneMinuteOnScreen.dispose();
super.dispose();
@@ -96,6 +105,9 @@ class _LoadingPhotosWidgetState extends State<LoadingPhotosWidget> {
@override
Widget build(BuildContext context) {
if (_importProgressEvent == null) {
_loadingMessage = S.of(context).loadingYourPhotos;
}
_setupLoadingMessages(context);
final isLightMode = Theme.of(context).brightness == Brightness.light;
return Scaffold(

4
mobile/lib/ui/home/start_backup_hook_widget.dart
View File

@@ -49,8 +49,8 @@ class StartBackupHookWidget extends StatelessWidget {
// ignore: unawaited_futures
routeToPage(
context,
BackupFolderSelectionPage(
buttonText: S.of(context).startBackup,
const BackupFolderSelectionPage(
isFirstBackup: true,
),
);
}

14
mobile/lib/ui/settings/backup/backup_folder_selection_page.dart
View File

@@ -19,14 +19,14 @@ import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
import 'package:photos/utils/dialog_util.dart';
class BackupFolderSelectionPage extends StatefulWidget {
final bool isFirstBackup;
final bool isOnboarding;
final String buttonText;
const BackupFolderSelectionPage({
required this.buttonText,
required this.isFirstBackup,
this.isOnboarding = false,
Key? key,
}) : super(key: key);
super.key,
});
@override
State<BackupFolderSelectionPage> createState() =>
@@ -173,7 +173,11 @@ class _BackupFolderSelectionPageState extends State<BackupFolderSelectionPage> {
: () async {
await updateFolderSettings();
},
child: Text(widget.buttonText),
child: Text(
widget.isFirstBackup
? S.of(context).startBackup
: S.of(context).backup,
),
),
),
widget.isOnboarding

4
mobile/lib/ui/settings/backup/backup_section_widget.dart
View File

@@ -42,8 +42,8 @@ class BackupSectionWidgetState extends State<BackupSectionWidget> {
onTap: () async {
await routeToPage(
context,
BackupFolderSelectionPage(
buttonText: S.of(context).backup,
const BackupFolderSelectionPage(
isFirstBackup: false,
),
);
},

4
mobile/lib/ui/viewer/search/tab_empty_state.dart
View File

@@ -38,8 +38,8 @@ class SearchTabEmptyState extends StatelessWidget {
// ignore: unawaited_futures
routeToPage(
context,
BackupFolderSelectionPage(
buttonText: S.of(context).backup,
const BackupFolderSelectionPage(
isFirstBackup: false,
),
);
},

7
web/apps/photos/src/components/Collections/CollectionCard.tsx
View File

@@ -1,11 +1,12 @@
import downloadManager from "@/new/photos/services/download";
import { EnteFile } from "@/new/photos/types/file";
import {
LoadingThumbnail,
StaticThumbnail,
} from "components/PlaceholderThumbnails";
} from "@/new/photos/components/PlaceholderThumbnails";
import downloadManager from "@/new/photos/services/download";
import { EnteFile } from "@/new/photos/types/file";
import { useEffect, useState } from "react";
/** See also: {@link ItemCard}. */
export default function CollectionCard(props: {
children?: any;
coverFile: EnteFile;

6
web/apps/photos/src/components/Collections/styledComponents.ts
View File

@@ -35,6 +35,7 @@ export const ScrollContainer = styled("div")`
gap: 4px;
`;
/** See also: {@link ItemTile}. */
export const CollectionTile = styled("div")`
display: flex;
position: relative;
@@ -67,11 +68,6 @@ export const AllCollectionTile = styled(CollectionTile)`
height: 150px;
`;
export const ResultPreviewTile = styled(CollectionTile)`
width: 48px;
height: 48px;
`;
export const CollectionBarTileText = styled(Overlay)`
padding: 4px;
background: linear-gradient(

2
web/apps/photos/src/components/ExportPendingList.tsx
View File

@@ -1,3 +1,4 @@
import { ResultPreviewTile } from "@/new/photos/components/ItemCards";
import { EnteFile } from "@/new/photos/types/file";
import { FlexWrapper } from "@ente/shared/components/Container";
import DialogBoxV2 from "@ente/shared/components/DialogBoxV2";
@@ -5,7 +6,6 @@ import { Box, styled } from "@mui/material";
import ItemList from "components/ItemList";
import { t } from "i18next";
import CollectionCard from "./Collections/CollectionCard";
import { ResultPreviewTile } from "./Collections/styledComponents";
interface Iprops {
isOpen: boolean;

8
web/apps/photos/src/components/pages/gallery/PreviewCard.tsx
View File

@@ -1,5 +1,9 @@
import log from "@/base/log";
import { FileType } from "@/media/file-type";
import {
LoadingThumbnail,
StaticThumbnail,
} from "@/new/photos/components/PlaceholderThumbnails";
import DownloadManager from "@/new/photos/services/download";
import { EnteFile } from "@/new/photos/types/file";
import { Overlay } from "@ente/shared/components/Container";
@@ -12,10 +16,6 @@ import {
GAP_BTW_TILES,
IMAGE_CONTAINER_MAX_WIDTH,
} from "components/PhotoList/constants";
import {
LoadingThumbnail,
StaticThumbnail,
} from "components/PlaceholderThumbnails";
import i18n from "i18next";
import { DeduplicateContext } from "pages/deduplicate";
import { GalleryContext } from "pages/gallery";

118
web/apps/photos/src/pages/gallery.tsx
View File

@@ -3,6 +3,10 @@ import { NavbarBase } from "@/base/components/Navbar";
import { useIsMobileWidth } from "@/base/hooks";
import log from "@/base/log";
import type { Collection } from "@/media/collection";
import {
SearchBar,
type SearchBarProps,
} from "@/new/photos/components/SearchBar";
import { WhatsNew } from "@/new/photos/components/WhatsNew";
import { shouldShowWhatsNew } from "@/new/photos/services/changelog";
import downloadManager from "@/new/photos/services/download";
@@ -15,10 +19,7 @@ import {
filterSearchableFiles,
setSearchableData,
} from "@/new/photos/services/search";
import {
SearchQuery,
SearchResultSummary,
} from "@/new/photos/services/search/types";
import type { SearchOption } from "@/new/photos/services/search/types";
import { EnteFile } from "@/new/photos/types/file";
import { mergeMetadata } from "@/new/photos/utils/file";
import {
@@ -74,7 +75,6 @@ import GalleryEmptyState from "components/GalleryEmptyState";
import { LoadingOverlay } from "components/LoadingOverlay";
import PhotoFrame from "components/PhotoFrame";
import { ITEM_TYPE, TimeStampListItem } from "components/PhotoList";
import { SearchBar, type UpdateSearch } from "components/SearchBar";
import Sidebar from "components/Sidebar";
import { type UploadTypeSelectorIntent } from "components/Upload/UploadTypeSelector";
import Uploader from "components/Upload/Uploader";
@@ -206,12 +206,8 @@ export default function Gallery() {
const [collectionNamerAttributes, setCollectionNamerAttributes] =
useState<CollectionNamerAttributes>(null);
const [collectionNamerView, setCollectionNamerView] = useState(false);
const [searchQuery, setSearchQuery] = useState<SearchQuery>(null);
const [shouldDisableDropzone, setShouldDisableDropzone] = useState(false);
const [isPhotoSwipeOpen, setIsPhotoSwipeOpen] = useState(false);
// TODO(MR): This is never true currently, this is the WIP ability to show
// what's new dialog on desktop app updates. The UI is done, need to hook
// this up to logic to trigger it.
const [openWhatsNew, setOpenWhatsNew] = useState(false);
const {
@@ -249,9 +245,12 @@ export default function Gallery() {
accept: ".zip",
});
// If we're in "search mode". See: [Note: "search mode"].
const [isInSearchMode, setIsInSearchMode] = useState(false);
const [searchResultSummary, setSetSearchResultSummary] =
useState<SearchResultSummary>(null);
// The option selected by the user selected from the search bar dropdown.
const [selectedSearchOption, setSelectedSearchOption] = useState<
SearchOption | undefined
>();
const syncInProgress = useRef(true);
const syncInterval = useRef<NodeJS.Timeout>();
const resync = useRef<{ force: boolean; silent: boolean }>();
@@ -492,18 +491,18 @@ export default function Gallery() {
}, [router.isReady]);
useEffect(() => {
if (isInSearchMode && searchResultSummary) {
if (isInSearchMode && selectedSearchOption) {
setPhotoListHeader({
height: 104,
item: (
<SearchResultSummaryHeader
searchResultSummary={searchResultSummary}
<SearchResultsHeader
selectedOption={selectedSearchOption}
/>
),
itemType: ITEM_TYPE.HEADER,
});
}
}, [isInSearchMode, searchResultSummary]);
}, [isInSearchMode, selectedSearchOption]);
const activeCollection = useMemo(() => {
if (!collections || !hiddenCollections) {
@@ -527,7 +526,7 @@ export default function Gallery() {
return;
}
if (activeCollectionID === TRASH_SECTION && !isInSearchMode) {
if (activeCollectionID === TRASH_SECTION && !selectedSearchOption) {
return getUniqueFiles([
...trashedFiles,
...files.filter((file) => tempDeletedFileIds?.has(file.id)),
@@ -535,8 +534,10 @@ export default function Gallery() {
}
let filteredFiles: EnteFile[] = [];
if (isInSearchMode) {
filteredFiles = await filterSearchableFiles(searchQuery);
if (selectedSearchOption) {
filteredFiles = await filterSearchableFiles(
selectedSearchOption.suggestion,
);
} else {
filteredFiles = getUniqueFiles(
(isInHiddenSection ? hiddenFiles : files).filter((item) => {
@@ -596,11 +597,6 @@ export default function Gallery() {
}),
);
}
if (searchQuery?.clip) {
return filteredFiles.sort((a, b) => {
return searchQuery.clip.get(b.id) - searchQuery.clip.get(a.id);
});
}
const sortAsc = activeCollection?.pubMagicMetadata?.data?.asc ?? false;
if (sortAsc) {
return sortFiles(filteredFiles, true);
@@ -614,7 +610,7 @@ export default function Gallery() {
tempDeletedFileIds,
tempHiddenFileIds,
hiddenFileIds,
searchQuery,
selectedSearchOption,
activeCollectionID,
archivedCollections,
]);
@@ -980,16 +976,18 @@ export default function Gallery() {
});
};
const updateSearch: UpdateSearch = (newSearch, summary) => {
if (newSearch?.collection) {
setActiveCollectionID(newSearch?.collection);
const handleSelectSearchOption = (
searchOption: SearchOption | undefined,
) => {
if (searchOption?.suggestion.type == "collection") {
setIsInSearchMode(false);
setSelectedSearchOption(undefined);
setActiveCollectionID(searchOption.suggestion.collectionID);
} else {
setSearchQuery(newSearch);
setIsInSearchMode(!!newSearch);
setSetSearchResultSummary(summary);
setIsInSearchMode(!!searchOption);
setSelectedSearchOption(searchOption);
}
setIsClipSearchResult(!!newSearch?.clip);
setIsClipSearchResult(searchOption?.suggestion.type == "clip");
};
const openUploader = (intent?: UploadTypeSelectorIntent) => {
@@ -1098,7 +1096,7 @@ export default function Gallery() {
openUploader={openUploader}
isInSearchMode={isInSearchMode}
setIsInSearchMode={setIsInSearchMode}
updateSearch={updateSearch}
onSelectSearchOption={handleSelectSearchOption}
/>
)}
</NavbarBase>
@@ -1271,29 +1269,20 @@ const mergeMaps = <K, V>(map1: Map<K, V>, map2: Map<K, V>) => {
return mergedMap;
};
interface NormalNavbarContentsProps {
type NormalNavbarContentsProps = SearchBarProps & {
openSidebar: () => void;
openUploader: () => void;
isInSearchMode: boolean;
setIsInSearchMode: (v: boolean) => void;
updateSearch: UpdateSearch;
}
};
const NormalNavbarContents: React.FC<NormalNavbarContentsProps> = ({
openSidebar,
openUploader,
isInSearchMode,
setIsInSearchMode,
updateSearch,
...props
}) => (
<>
{!isInSearchMode && <SidebarButton onClick={openSidebar} />}
<SearchBar
isInSearchMode={isInSearchMode}
setIsInSearchMode={setIsInSearchMode}
updateSearch={updateSearch}
/>
{!isInSearchMode && <UploadButton onClick={openUploader} />}
{!props.isInSearchMode && <SidebarButton onClick={openSidebar} />}
<SearchBar {...props} />
{!props.isInSearchMode && <UploadButton onClick={openUploader} />}
</>
);
@@ -1352,25 +1341,20 @@ const HiddenSectionNavbarContents: React.FC<
</HorizontalFlex>
);
interface SearchResultSummaryHeaderProps {
searchResultSummary: SearchResultSummary;
interface SearchResultsHeaderProps {
selectedOption: SearchOption;
}
const SearchResultSummaryHeader: React.FC<SearchResultSummaryHeaderProps> = ({
searchResultSummary,
}) => {
if (!searchResultSummary) {
return <></>;
}
const { optionName, fileCount } = searchResultSummary;
return (
<CollectionInfoBarWrapper>
<Typography color="text.muted" variant="large">
{t("search_results")}
</Typography>
<CollectionInfo name={optionName} fileCount={fileCount} />
</CollectionInfoBarWrapper>
);
};
const SearchResultsHeader: React.FC<SearchResultsHeaderProps> = ({
selectedOption,
}) => (
<CollectionInfoBarWrapper>
<Typography color="text.muted" variant="large">
{t("search_results")}
</Typography>
<CollectionInfo
name={selectedOption.suggestion.label}
fileCount={selectedOption.fileCount}
/>
</CollectionInfoBarWrapper>
);

0
web/apps/photos/src/types/image/index.ts
View File

0
web/apps/photos/src/types/search/index.ts
View File

69
web/packages/new/photos/components/ItemCards.tsx Normal file
View File

@@ -0,0 +1,69 @@
import {
LoadingThumbnail,
StaticThumbnail,
} from "@/new/photos/components/PlaceholderThumbnails";
import downloadManager from "@/new/photos/services/download";
import { type EnteFile } from "@/new/photos/types/file";
import { styled } from "@mui/material";
import React, { useEffect, useState } from "react";
interface ItemCardProps {
/** The file whose thumbnail (if any) should be should be shown. */
coverFile: EnteFile;
/** One of the *Tile components to use as the top level element. */
TileComponent: React.FC<React.PropsWithChildren>;
}
/**
* A simplified variant of {@link CollectionCard}, meant to be used for
* representing either collections and files.
*/
export const ItemCard: React.FC<ItemCardProps> = ({
coverFile,
TileComponent,
}) => {
const [coverImageURL, setCoverImageURL] = useState("");
useEffect(() => {
const main = async () => {
const url = await downloadManager.getThumbnailForPreview(coverFile);
if (url) setCoverImageURL(url);
};
void main();
}, [coverFile]);
return (
<TileComponent>
{coverFile.metadata.hasStaticThumbnail ? (
<StaticThumbnail fileType={coverFile.metadata.fileType} />
) : coverImageURL ? (
<img src={coverImageURL} />
) : (
<LoadingThumbnail />
)}
</TileComponent>
);
};
/**
* A verbatim copy of CollectionTile, meant to be used with ItemCards.
*/
export const ItemTile = styled("div")`
display: flex;
position: relative;
border-radius: 4px;
overflow: hidden;
cursor: pointer;
& > img {
object-fit: cover;
width: 100%;
height: 100%;
pointer-events: none;
}
user-select: none;
`;
export const ResultPreviewTile = styled(ItemTile)`
width: 48px;
height: 48px;
`;

8
web/packages/new/photos/components/PeopleList.tsx
View File

@@ -1,14 +1,14 @@
import type { Person } from "@/new/photos/services/ml";
import { faceCrop, unidentifiedFaceIDs } from "@/new/photos/services/ml";
import type { EnteFile } from "@/new/photos/types/file";
import { Skeleton, Typography, styled } from "@mui/material";
import { t } from "i18next";
import React, { useEffect, useState } from "react";
import type { SearchPerson } from "../services/search/types";
export interface PeopleListProps {
people: SearchPerson[];
people: Person[];
maxRows: number;
onSelect?: (person: SearchPerson, index: number) => void;
onSelect?: (person: Person, index: number) => void;
}
export const PeopleList: React.FC<PeopleListProps> = ({
@@ -60,7 +60,7 @@ const FaceChip = styled("div")<{ clickable?: boolean }>`
export interface PhotoPeopleListProps {
file: EnteFile;
onSelect?: (person: SearchPerson, index: number) => void;
onSelect?: (person: Person, index: number) => void;
}
export function PhotoPeopleList() {

3
web/apps/photos/src/components/PlaceholderThumbnails.tsx → web/packages/new/photos/components/PlaceholderThumbnails.tsx
View File

@@ -3,6 +3,7 @@ import { Overlay } from "@ente/shared/components/Container";
import PhotoOutlined from "@mui/icons-material/PhotoOutlined";
import PlayCircleOutlineOutlined from "@mui/icons-material/PlayCircleOutlineOutlined";
import { styled } from "@mui/material";
import React from "react";
interface Iprops {
fileType: FileType;
@@ -14,7 +15,7 @@ const CenteredOverlay = styled(Overlay)`
align-items: center;
`;
export const StaticThumbnail = (props: Iprops) => {
export const StaticThumbnail: React.FC<Iprops> = (props) => {
return (
<CenteredOverlay
sx={(theme) => ({

333
web/apps/photos/src/components/SearchBar.tsx → web/packages/new/photos/components/SearchBar.tsx
View File

@@ -1,33 +1,16 @@
import { assertionFailed } from "@/base/assert";
import { useIsMobileWidth } from "@/base/hooks";
import { FileType } from "@/media/file-type";
import { ItemCard, ResultPreviewTile } from "@/new/photos/components/ItemCards";
import {
isMLSupported,
mlStatusSnapshot,
mlStatusSubscribe,
} from "@/new/photos/services/ml";
import { getAutoCompleteSuggestions } from "@/new/photos/services/search";
import type {
City,
SearchDateComponents,
SearchPerson,
SearchResultSummary,
} from "@/new/photos/services/search/types";
import {
ClipSearchScores,
SearchOption,
SearchQuery,
SuggestionType,
} from "@/new/photos/services/search/types";
import { labelForSuggestionType } from "@/new/photos/services/search/ui";
import type { LocationTag } from "@/new/photos/services/user-entity";
import {
FreeFlowText,
SpaceBetweenFlex,
} from "@ente/shared/components/Container";
import { searchOptionsForString } from "@/new/photos/services/search";
import type { SearchOption } from "@/new/photos/services/search/types";
import { nullToUndefined } from "@/utils/transform";
import CalendarIcon from "@mui/icons-material/CalendarMonth";
import CloseIcon from "@mui/icons-material/Close";
import FolderIcon from "@mui/icons-material/Folder";
import ImageIcon from "@mui/icons-material/Image";
import LocationIcon from "@mui/icons-material/LocationOn";
import SearchIcon from "@mui/icons-material/Search";
@@ -41,29 +24,21 @@ import {
useTheme,
type Theme,
} from "@mui/material";
import CollectionCard from "components/Collections/CollectionCard";
import { ResultPreviewTile } from "components/Collections/styledComponents";
import { t } from "i18next";
import pDebounce from "p-debounce";
import {
useCallback,
useEffect,
useMemo,
useRef,
useState,
useSyncExternalStore,
} from "react";
import React, { useMemo, useRef, useState, useSyncExternalStore } from "react";
import {
components as SelectComponents,
type ControlProps,
type InputActionMeta,
type InputProps,
type OptionProps,
type SelectInstance,
type StylesConfig,
} from "react-select";
import AsyncSelect from "react-select/async";
interface SearchBarProps {
export interface SearchBarProps {
/**
* [Note: "Search mode"]
*
@@ -76,22 +51,20 @@ interface SearchBarProps {
*
* When we're in search mode,
*
* 1. Other icons from the navbar are hidden
* 1. Other icons from the navbar are hidden.
* 2. Next to the search input there is a cancel button to exit search mode.
*/
isInSearchMode: boolean;
/**
* Enter or exit "search mode".
*/
setIsInSearchMode: (v: boolean) => void;
updateSearch: UpdateSearch;
setIsInSearchMode: (b: boolean) => void;
/**
* Set or clear the selected {@link SearchOption}.
*/
onSelectSearchOption: (o: SearchOption | undefined) => void;
}
export type UpdateSearch = (
search: SearchQuery,
summary: SearchResultSummary,
) => void;
/**
* The search bar is a styled "select" element that allow the user to type in
* the attached input field, and shows a list of matching suggestions in a
@@ -110,7 +83,7 @@ export type UpdateSearch = (
export const SearchBar: React.FC<SearchBarProps> = ({
setIsInSearchMode,
isInSearchMode,
...props
onSelectSearchOption,
}) => {
const isMobileWidth = useIsMobileWidth();
@@ -121,7 +94,7 @@ export const SearchBar: React.FC<SearchBarProps> = ({
{isMobileWidth && !isInSearchMode ? (
<MobileSearchArea onSearch={showSearchInput} />
) : (
<SearchInput {...props} isInSearchMode={isInSearchMode} />
<SearchInput {...{ isInSearchMode, onSelectSearchOption }} />
)}
</Box>
);
@@ -140,112 +113,77 @@ const MobileSearchArea: React.FC<MobileSearchAreaProps> = ({ onSearch }) => (
</Box>
);
interface SearchInputProps {
isInSearchMode: boolean;
updateSearch: UpdateSearch;
}
const SearchInput: React.FC<SearchInputProps> = ({
const SearchInput: React.FC<Omit<SearchBarProps, "setIsInSearchMode">> = ({
isInSearchMode,
updateSearch,
onSelectSearchOption,
}) => {
// A ref to the top level Select.
const selectRef = useRef(null);
const selectRef = useRef<SelectInstance<SearchOption> | null>(null);
// The currently selected option.
const [value, setValue] = useState<SearchOption | undefined>();
//
// We need to use `null` instead of `undefined` to indicate missing values,
// because using `undefined` instead moves the Select from being a controlled
// component to an uncontrolled component.
const [value, setValue] = useState<SearchOption | null>(null);
// The contents of the input field associated with the select.
const [inputValue, setInputValue] = useState("");
const theme = useTheme();
const styles = useMemo(() => useSelectStyles(theme), [theme]);
const styles = useMemo(() => createSelectStyles(theme), [theme]);
const components = useMemo(() => ({ Control, Input, Option }), []);
useEffect(() => {
search(value);
}, [value]);
const handleChange = (value: SearchOption | null) => {
// Collection suggestions are handled differently - our caller will
// switch to the collection view, dismissing search.
if (value?.suggestion.type == "collection") {
setValue(null);
setInputValue("");
} else {
setValue(value);
setInputValue(value?.suggestion.label ?? "");
}
// Let our parent know the selection was changed.
onSelectSearchOption(nullToUndefined(value));
const handleChange = (value: SearchOption) => {
setValue(value);
setInputValue(value?.label);
// The Select has a blurInputOnSelect prop, but that makes the input
// field lose focus, not the entire menu (e.g. when pressing twice).
//
// We anyways need the ref so that we can blur on selecting a person
// from the default options.
// from the default options. So also use it to blur the entire Select
// (including the menu) when the user selects an option.
selectRef.current?.blur();
};
const handleInputChange = (value: string, actionMeta: InputActionMeta) => {
if (actionMeta.action === "input-change") {
setInputValue(value);
}
if (actionMeta.action == "input-change") setInputValue(value);
};
const resetSearch = () => {
updateSearch(null, null);
setValue(null);
setInputValue("");
};
const getOptions = useCallback(
pDebounce(getAutoCompleteSuggestions(), 250),
[],
);
const search = (selectedOption: SearchOption) => {
if (!selectedOption) {
return;
}
let search: SearchQuery;
switch (selectedOption.type) {
case SuggestionType.DATE:
search = {
date: selectedOption.value as SearchDateComponents,
};
break;
case SuggestionType.LOCATION:
search = {
location: selectedOption.value as LocationTag,
};
break;
case SuggestionType.CITY:
search = {
city: selectedOption.value as City,
};
break;
case SuggestionType.COLLECTION:
search = { collection: selectedOption.value as number };
setValue(null);
setInputValue("");
break;
case SuggestionType.FILE_NAME:
search = { files: selectedOption.value as number[] };
break;
case SuggestionType.FILE_CAPTION:
search = { files: selectedOption.value as number[] };
break;
case SuggestionType.PERSON:
search = { person: selectedOption.value as SearchPerson };
break;
case SuggestionType.FILE_TYPE:
search = { fileType: selectedOption.value as FileType };
break;
case SuggestionType.CLIP:
search = { clip: selectedOption.value as ClipSearchScores };
}
updateSearch(search, {
optionName: selectedOption.label,
fileCount: selectedOption.fileCount,
});
onSelectSearchOption(undefined);
};
const handleSelectCGroup = (value: SearchOption) => {
// Dismiss the search menu.
selectRef.current?.blur();
setValue(value);
onSelectSearchOption(undefined);
};
const components = useMemo(() => ({ Option, Control, Input }), []);
const handleFocus = () => {
// A workaround to show the suggestions again for the current non-empty
// search string if the user focuses back on the input field after
// moving focus elsewhere.
if (inputValue) {
selectRef.current?.onInputChange(inputValue, {
action: "set-value",
prevInputValue: "",
});
}
};
return (
<SearchInputWrapper>
@@ -254,14 +192,14 @@ const SearchInput: React.FC<SearchInputProps> = ({
value={value}
components={components}
styles={styles}
placeholder={t("search_hint")}
loadOptions={getOptions}
loadOptions={loadOptions}
onChange={handleChange}
isMulti={false}
isClearable
escapeClearsValue
inputValue={inputValue}
onInputChange={handleInputChange}
isClearable
escapeClearsValue
onFocus={handleFocus}
placeholder={t("search_hint")}
noOptionsMessage={({ inputValue }) =>
shouldShowEmptyState(inputValue) ? (
<EmptyState onSelectCGroup={handleSelectCGroup} />
@@ -270,7 +208,7 @@ const SearchInput: React.FC<SearchInputProps> = ({
/>
{isInSearchMode && (
<IconButton onClick={() => resetSearch()} sx={{ ml: 1 }}>
<IconButton onClick={resetSearch}>
<CloseIcon />
</IconButton>
)}
@@ -283,12 +221,15 @@ const SearchInputWrapper = styled(Box)`
width: 100%;
align-items: center;
justify-content: center;
gap: 8px;
background: ${({ theme }) => theme.colors.background.base};
max-width: 484px;
margin: auto;
`;
const useSelectStyles = ({
const loadOptions = pDebounce(searchOptionsForString, 250);
const createSelectStyles = ({
colors,
}: Theme): StylesConfig<SearchOption, false> => ({
container: (style) => ({ ...style, flex: 1 }),
@@ -316,9 +257,9 @@ const useSelectStyles = ({
"& :hover": {
cursor: "pointer",
},
"& .main": {
backgroundColor: isFocused && colors.background.elevated2,
},
"& .option-contents": isFocused
? { backgroundColor: colors.background.elevated2 }
: {},
"&:last-child .MuiDivider-root": {
display: "none",
},
@@ -353,29 +294,37 @@ const Control = ({ children, ...props }: ControlProps<SearchOption, false>) => (
color: (theme) => theme.colors.stroke.muted,
}}
>
{iconForOptionType(props.getValue()[0]?.type)}
{iconForOption(props.getValue()[0])}
</Box>
{children}
</Stack>
</SelectComponents.Control>
);
const iconForOptionType = (type: SuggestionType | undefined) => {
switch (type) {
case SuggestionType.DATE:
return <CalendarIcon />;
case SuggestionType.LOCATION:
case SuggestionType.CITY:
return <LocationIcon />;
case SuggestionType.COLLECTION:
return <FolderIcon />;
case SuggestionType.FILE_NAME:
const iconForOption = (option: SearchOption | undefined) => {
switch (option?.suggestion.type) {
case "fileName":
return <ImageIcon />;
case "date":
return <CalendarIcon />;
case "location":
case "city":
return <LocationIcon />;
default:
return <SearchIcon />;
}
};
/**
* A custom input for react-select that is always visible.
*
* This is a workaround to allow the search string to be always displayed, and
* editable, even after the user has moved focus away from it.
*/
const Input: React.FC<InputProps<SearchOption, false>> = (props) => (
<SelectComponents.Input {...props} isHidden={false} />
);
/**
* A preflight check for whether or not we should show the EmptyState.
*
@@ -391,7 +340,7 @@ const shouldShowEmptyState = (inputValue: string) => {
// Don't show empty state if there is no ML related information.
if (!isMLSupported) return false;
const status = isMLSupported && mlStatusSnapshot();
const status = mlStatusSnapshot();
if (!status || status.phase == "disabled") return false;
// Show it otherwise.
@@ -434,6 +383,9 @@ const EmptyState: React.FC<EmptyStateProps> = () => {
break;
}
// TODO-Cluster this is where it'll go.
// const people = wipPersons();
return (
<Box>
<Typography variant="mini" sx={{ textAlign: "left" }}>
@@ -447,7 +399,7 @@ const EmptyState: React.FC<EmptyStateProps> = () => {
// const peopleSuggestions = options.filter(
// (o) => o.type === SuggestionType.PERSON,
// );
// const people = peopleSuggestions.map((o) => o.value as SearchPerson);
// const people = peopleSuggestions.map((o) => o.value as Person);
// return (
// <SelectComponents.Menu {...props}>
// <Box my={1}>
@@ -508,7 +460,7 @@ export async function getAllPeopleSuggestion(): Promise<Array<Suggestion>> {
}
async function getAllPeople(limit: number = undefined) {
return (await wipSearchPersons()).slice(0, limit);
return (await wipPersons()).slice(0, limit);
// TODO-Clustetr
// if (done) return [];
@@ -519,7 +471,7 @@ async function getAllPeople(limit: number = undefined) {
// log.debug(() => ["people", { people }]);
// }
// let people: Array<SearchPerson> = []; // await mlIDbStorage.getAllPeople();
// let people: Array<Person> = []; // await mlIDbStorage.getAllPeople();
// people = await wipCluster();
// // await mlPeopleStore.iterate<Person, void>((person) => {
// // people.push(person);
@@ -532,54 +484,73 @@ async function getAllPeople(limit: number = undefined) {
// return result;
}
*/
const Option: React.FC<OptionProps<SearchOption, false>> = (props) => (
<SelectComponents.Option {...props}>
<LabelWithInfo data={props.data} />
<OptionContents data={props.data} />
<Divider sx={{ mx: 2, my: 1 }} />
</SelectComponents.Option>
);
const LabelWithInfo = ({ data }: { data: SearchOption }) => {
return (
<>
<Box className="main" px={2} py={1}>
<Typography variant="mini" mb={1}>
{labelForSuggestionType(data.type)}
const OptionContents = ({ data: option }: { data: SearchOption }) => (
<Stack className="option-contents" gap={1} px={2} py={1}>
<Typography variant="mini">{labelForOption(option)}</Typography>
<Stack
direction="row"
gap={1}
sx={{ alignItems: "center", justifyContent: "space-between" }}
>
<Box>
<Typography
sx={{ fontWeight: "bold", wordBreak: "break-word" }}
>
{option.suggestion.label}
</Typography>
<Typography color="text.muted">
{t("photos_count", { count: option.fileCount })}
</Typography>
<SpaceBetweenFlex>
<Box mr={1}>
<FreeFlowText>
<Typography fontWeight={"bold"}>
{data.label}
</Typography>
</FreeFlowText>
<Typography color="text.muted">
{t("photos_count", { count: data.fileCount })}
</Typography>
</Box>
<Stack direction={"row"} spacing={1}>
{data.previewFiles.map((file) => (
<CollectionCard
key={file.id}
coverFile={file}
onClick={() => null}
collectionTile={ResultPreviewTile}
/>
))}
</Stack>
</SpaceBetweenFlex>
</Box>
<Divider sx={{ mx: 2, my: 1 }} />
</>
);
};
// A custom input for react-select that is always visible. This is a roundabout
// hack the existing code used to display the search string when showing the
// results page; likely there should be a better way.
const Input: React.FC<InputProps<SearchOption, false>> = (props) => (
<SelectComponents.Input {...props} isHidden={false} />
<Stack direction={"row"} gap={1}>
{option.previewFiles.map((file) => (
<ItemCard
key={file.id}
coverFile={file}
TileComponent={ResultPreviewTile}
/>
))}
</Stack>
</Stack>
</Stack>
);
const labelForOption = (option: SearchOption) => {
switch (option.suggestion.type) {
case "collection":
return t("album");
case "fileType":
return t("file_type");
case "fileName":
return t("file_name");
case "fileCaption":
return t("description");
case "date":
return t("date");
case "location":
return t("location");
case "city":
return t("location");
case "clip":
return t("magic");
case "person":
return t("person");
}
};

138
web/packages/new/photos/services/ml/index.ts
View File

@@ -17,7 +17,6 @@ import { throttled } from "@/utils/promise";
import { proxy, transfer } from "comlink";
import { isInternalUser } from "../feature-flags";
import { getRemoteFlag, updateRemoteFlag } from "../remote-store";
import type { SearchPerson } from "../search/types";
import type { UploadItem } from "../upload/types";
import {
type ClusterFace,
@@ -59,19 +58,31 @@ class MLState {
comlinkWorker: Promise<ComlinkWorker<typeof MLWorker>> | undefined;
/**
* Subscriptions to {@link MLStatus}.
* Subscriptions to {@link MLStatus} updates.
*
* See {@link mlStatusSubscribe}.
*/
mlStatusListeners: (() => void)[] = [];
/**
* Snapshot of {@link MLStatus}.
*
* See {@link mlStatusSnapshot}.
* Snapshot of the {@link MLStatus} returned by the {@link mlStatusSnapshot}
* function.
*/
mlStatusSnapshot: MLStatus | undefined;
/**
* Subscriptions to updates to the list of {@link Person}s we know about.
*
* See {@link peopleSubscribe}.
*/
peopleListeners: (() => void)[] = [];
/**
* Snapshot of the {@link Person}s returned by the {@link peopleSnapshot}
* function.
*/
peopleSnapshot: Person[] | undefined;
/**
* In flight face crop regeneration promises indexed by the IDs of the files
* whose faces we are regenerating.
@@ -329,7 +340,7 @@ export const wipClusterEnable = async (): Promise<boolean> =>
// // TODO-Cluster temporary state here
let _wip_isClustering = false;
let _wip_searchPersons: SearchPerson[] | undefined;
let _wip_people: Person[] | undefined;
let _wip_hasSwitchedOnce = false;
export const wipHasSwitchedOnceCmpAndSet = () => {
@@ -338,11 +349,6 @@ export const wipHasSwitchedOnceCmpAndSet = () => {
return false;
};
export const wipSearchPersons = async () => {
if (!(await wipClusterEnable())) return [];
return _wip_searchPersons ?? [];
};
export interface ClusterPreviewWithFile {
clusterSize: number;
faces: ClusterPreviewFaceWithFile[];
@@ -374,7 +380,7 @@ export const wipClusterDebugPageContents = async (
log.info("clustering", opts);
_wip_isClustering = true;
_wip_searchPersons = undefined;
_wip_people = undefined;
triggerStatusUpdate();
const {
@@ -407,7 +413,7 @@ export const wipClusterDebugPageContents = async (
const clusterByID = new Map(clusters.map((c) => [c.id, c]));
const searchPersons = cgroups
const people = cgroups
.map((cgroup) => {
const faceID = ensure(cgroup.displayFaceID);
const fileID = ensure(fileIDFromFaceID(faceID));
@@ -432,7 +438,7 @@ export const wipClusterDebugPageContents = async (
.sort((a, b) => b.faceIDs.length - a.faceIDs.length);
_wip_isClustering = false;
_wip_searchPersons = searchPersons;
_wip_people = people;
triggerStatusUpdate();
return {
@@ -514,7 +520,7 @@ export const mlStatusSnapshot = (): MLStatus | undefined => {
*/
const triggerStatusUpdate = () => void updateMLStatusSnapshot();
/** Unconditionally update of the {@link MLStatus} snapshot. */
/** Unconditional update of the {@link MLStatus} snapshot. */
const updateMLStatusSnapshot = async () =>
setMLStatusSnapshot(await getMLStatus());
@@ -581,13 +587,111 @@ const setInterimScheduledStatus = () => {
const workerDidProcessFileOrIdle = throttled(updateMLStatusSnapshot, 2000);
/**
* A massaged version of {@link CGroup} suitable for being shown in the UI.
*
* While cgroups are synced with remote, they do not directly correspond to
* "people" (this is ignoring the other issue that the cluster groups may be for
* non-human faces too). CGroups represent both positive and negative feedback,
* and the negations are specifically meant so that they're not shown in the UI.
*
* So while each person has an underlying cgroups, not all cgroups have a
* corresponding person.
*
* Beyond this, a {@link Person} object has data converted into a format that
* the UI can use directly and efficiently (as compared to a {@link CGroup},
* which is tailored for transmission and storage).
*/
export interface Person {
/** Unique ID (nanoid) of the underlying {@link CGroup}. */
id: string;
/** If this is a named person, then their name. */
name?: string;
/** The files in which this face occurs. */
files: number[];
/**
* The face that should be used as the "cover" face to represent this
* {@link Person} in the UI.
*/
displayFaceID: string;
/**
* The {@link EnteFile} which contains the display face.
*/
displayFaceFile: EnteFile;
}
/**
* A function that can be used to subscribe to updates to {@link Person}s.
*
* This, along with {@link peopleSnapshot}, is meant to be used as arguments to
* React's {@link useSyncExternalStore}.
*
* @param callback A function that will be invoked whenever the result of
* {@link peopleSnapshot} changes.
*
* @returns A function that can be used to clear the subscription.
*/
export const peopleSubscribe = (onChange: () => void): (() => void) => {
_state.peopleListeners.push(onChange);
return () => {
_state.peopleListeners = _state.peopleListeners.filter(
(l) => l != onChange,
);
};
};
/**
* Return the last known, cached {@link people}.
*
* This, along with {@link peopleSnapshot}, is meant to be used as arguments to
* React's {@link useSyncExternalStore}.
*
* A return value of `undefined` indicates that we're either still loading the
* initial list of people, or that the user has ML disabled and thus doesn't
* have any people (this is distinct from the case where the user has ML enabled
* but doesn't have any named "person" clusters so far).
*/
export const peopleSnapshot = (): Person[] | undefined => {
const result = _state.peopleSnapshot;
// We don't have it yet, trigger an update.
if (!result) triggerPeopleUpdate();
return result;
};
/**
* Trigger an asynchronous and unconditional update of the people snapshot.
*/
const triggerPeopleUpdate = () => void updatePeopleSnapshot();
/** Unconditional update of the people snapshot. */
const updatePeopleSnapshot = async () => setPeopleSnapshot(await getPeople());
const setPeopleSnapshot = (snapshot: Person[] | undefined) => {
_state.peopleSnapshot = snapshot;
_state.peopleListeners.forEach((l) => l());
};
/**
* Compute the list of people.
*
* TODO-Cluster this is a placeholder function and might not be needed since
* people might be updated in a push based manner.
*/
const getPeople = async (): Promise<Person[] | undefined> => {
if (!_state.isMLEnabled) return undefined;
// TODO-Cluster additional check for now as it is heavily WIP.
if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return undefined;
if (!(await wipClusterEnable())) return [];
return _wip_people;
};
/**
* Use CLIP to perform a natural language search over image embeddings.
*
* @param searchPhrase Normalized (trimmed and lowercased) search phrase.
*
* It returns file (IDs) that should be shown in the search results, along with
* their scores.
* It returns file (IDs) that should be shown in the search results, each
* annotated with its score.
*
* The result can also be `undefined`, which indicates that the download for the
* ML model is still in progress (trying again later should succeed).

148
web/packages/new/photos/services/search/index.ts
View File

@@ -1,25 +1,16 @@
import { isDesktop } from "@/base/app";
import log from "@/base/log";
import { masterKeyFromSession } from "@/base/session-store";
import { ComlinkWorker } from "@/base/worker/comlink-worker";
import { FileType } from "@/media/file-type";
import type { LocationTag } from "@/new/photos/services/user-entity";
import i18n, { t } from "i18next";
import { clipMatches, isMLEnabled } from "../ml";
import { clipMatches, isMLEnabled, isMLSupported } from "../ml";
import type {
City,
ClipSearchScores,
DateSearchResult,
LabelledFileType,
LabelledSearchDateComponents,
LocalizedSearchData,
SearchableData,
SearchDateComponents,
SearchOption,
SearchPerson,
SearchQuery,
Suggestion,
SearchSuggestion,
} from "./types";
import { SuggestionType } from "./types";
import type { SearchWorker } from "./worker";
/**
@@ -66,12 +57,23 @@ export const setSearchableData = (data: SearchableData) =>
void worker().then((w) => w.setSearchableData(data));
/**
* Convert a search string into a suggestions that can be shown in the search
* results, and can also be used filter the searchable files.
* Convert a search string into (annotated) suggestions that can be shown in the
* search results dropdown.
*
* @param searchString The string we want to search for.
*/
export const suggestionsForString = async (searchString: string) => {
export const searchOptionsForString = async (searchString: string) => {
const t = Date.now();
const suggestions = await suggestionsForString(searchString);
const options = await suggestionsToOptions(suggestions);
log.debug(() => [
"search",
{ searchString, options, duration: `${Date.now() - t} ms` },
]);
return options;
};
const suggestionsForString = async (searchString: string) => {
// Normalize it by trimming whitespace and converting to lowercase.
const s = searchString.trim().toLowerCase();
if (s.length == 0) return [];
@@ -80,7 +82,7 @@ export const suggestionsForString = async (searchString: string) => {
// separately, in parallel with the rest of the search query construction in
// the search worker, then combine the two.
const results = await Promise.all([
clipSuggestions(s, searchString).then((s) => s ?? []),
clipSuggestion(s, searchString).then((s) => s ?? []),
worker().then((w) =>
w.suggestionsForString(s, searchString, localizedSearchData()),
),
@@ -88,26 +90,44 @@ export const suggestionsForString = async (searchString: string) => {
return results.flat();
};
const clipSuggestions = async (s: string, searchString: string) => {
if (!isDesktop) return undefined;
const clipSuggestion = async (
s: string,
searchString: string,
): Promise<SearchSuggestion | undefined> => {
if (!isMLSupported) return undefined;
if (!isMLEnabled()) return undefined;
const matches = await clipMatches(s);
if (!matches) return undefined;
return {
type: SuggestionType.CLIP,
value: matches,
label: searchString,
};
return { type: "clip", clipScoreForFileID: matches, label: searchString };
};
const suggestionsToOptions = (suggestions: SearchSuggestion[]) =>
filterSearchableFilesMulti(suggestions).then((res) =>
res.map(([files, suggestion]) => ({
suggestion,
fileCount: files.length,
previewFiles: files.slice(0, 3),
})),
);
/**
* Return the list of {@link EnteFile}s (from amongst the previously set
* {@link SearchableData}) that match the given search {@link suggestion}.
*/
export const filterSearchableFiles = async (suggestion: SearchQuery) =>
export const filterSearchableFiles = async (suggestion: SearchSuggestion) =>
worker().then((w) => w.filterSearchableFiles(suggestion));
/**
* A batched variant of {@link filterSearchableFiles}.
*
* This has drastically (10x) better performance when filtering files for a
* large number of suggestions (e.g. single letter searches that lead to a large
* number of city prefix matches), likely because of reduced worker IPC.
*/
const filterSearchableFilesMulti = async (suggestions: SearchSuggestion[]) =>
worker().then((w) => w.filterSearchableFilesMulti(suggestions));
/**
* Cached value of {@link localizedSearchData}.
*/
@@ -144,7 +164,7 @@ const localizedSearchData = () =>
/**
* A list of holidays - their yearly dates and localized names.
*/
const holidays = (): DateSearchResult[] => [
const holidays = (): LabelledSearchDateComponents[] => [
{ components: { month: 12, day: 25 }, label: t("CHRISTMAS") },
{ components: { month: 12, day: 24 }, label: t("CHRISTMAS_EVE") },
{ components: { month: 1, day: 1 }, label: t("NEW_YEAR") },
@@ -159,81 +179,3 @@ const labelledFileTypes = (): LabelledFileType[] => [
{ fileType: FileType.video, label: t("VIDEO") },
{ fileType: FileType.livePhoto, label: t("LIVE_PHOTO") },
];
// TODO-Cluster -- AUDIT BELOW THIS
// Suggestions shown in the search dropdown when the user has typed something.
export const getAutoCompleteSuggestions =
() =>
async (searchPhrase: string): Promise<SearchOption[]> => {
log.debug(() => ["getAutoCompleteSuggestions"]);
try {
const suggestions: Suggestion[] =
await suggestionsForString(searchPhrase);
return convertSuggestionsToOptions(suggestions);
} catch (e) {
log.error("getAutoCompleteSuggestions failed", e);
return [];
}
};
async function convertSuggestionsToOptions(
suggestions: Suggestion[],
): Promise<SearchOption[]> {
const previewImageAppendedOptions: SearchOption[] = [];
for (const suggestion of suggestions) {
const searchQuery = convertSuggestionToSearchQuery(suggestion);
const resultFiles = await filterSearchableFiles(searchQuery);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (searchQuery?.clip) {
resultFiles.sort((a, b) => {
const aScore = searchQuery.clip?.get(a.id) ?? 0;
const bScore = searchQuery.clip?.get(b.id) ?? 0;
return bScore - aScore;
});
}
if (resultFiles.length) {
previewImageAppendedOptions.push({
...suggestion,
fileCount: resultFiles.length,
previewFiles: resultFiles.slice(0, 3),
});
}
}
return previewImageAppendedOptions;
}
function convertSuggestionToSearchQuery(option: Suggestion): SearchQuery {
switch (option.type) {
case SuggestionType.DATE:
return {
date: option.value as SearchDateComponents,
};
case SuggestionType.LOCATION:
return {
location: option.value as LocationTag,
};
case SuggestionType.CITY:
return { city: option.value as City };
case SuggestionType.COLLECTION:
return { collection: option.value as number };
case SuggestionType.FILE_NAME:
return { files: option.value as number[] };
case SuggestionType.FILE_CAPTION:
return { files: option.value as number[] };
case SuggestionType.PERSON:
return { person: option.value as SearchPerson };
case SuggestionType.FILE_TYPE:
return { fileType: option.value as FileType };
case SuggestionType.CLIP:
return { clip: option.value as ClipSearchScores };
}
}

120
web/packages/new/photos/services/search/types.ts
View File

@@ -6,9 +6,44 @@
import type { Location } from "@/base/types";
import type { Collection } from "@/media/collection";
import { FileType } from "@/media/file-type";
import type { Person } from "@/new/photos/services/ml";
import type { EnteFile } from "@/new/photos/types/file";
import type { LocationTag } from "../user-entity";
/**
* A search suggestion.
*
* These (wrapped up in {@link SearchOption}s) are shown in the search results
* dropdown, and can also be used to filter the list of files that are shown.
*/
export type SearchSuggestion = { label: string } & (
| { type: "collection"; collectionID: number }
| { type: "fileType"; fileType: FileType }
| { type: "fileName"; fileIDs: number[] }
| { type: "fileCaption"; fileIDs: number[] }
| { type: "date"; dateComponents: SearchDateComponents }
| { type: "location"; locationTag: LocationTag }
| { type: "city"; city: City }
| { type: "clip"; clipScoreForFileID: Map<number, number> }
| { type: "person"; person: Person }
);
/**
* An option shown in the the search bar's select dropdown.
*
* The {@link SearchOption} wraps a {@link SearchSuggestion} with some metadata
* used when showing a corresponding entry in the dropdown, and in the results
* header.
*
* If the user selects the option, then we will re-run the search using the
* {@link suggestion} to filter the list of files shown to the user.
*/
export interface SearchOption {
suggestion: SearchSuggestion;
fileCount: number;
previewFiles: EnteFile[];
}
/**
* The base data over which we should search.
*/
@@ -17,7 +52,7 @@ export interface SearchableData {
files: EnteFile[];
}
export interface DateSearchResult {
export interface LabelledSearchDateComponents {
components: SearchDateComponents;
label: string;
}
@@ -47,7 +82,7 @@ export type Searchable<T> = T & {
*/
export interface LocalizedSearchData {
locale: string;
holidays: Searchable<DateSearchResult>[];
holidays: Searchable<LabelledSearchDateComponents>[];
labelledFileTypes: Searchable<LabelledFileType>[];
}
@@ -83,18 +118,6 @@ export interface SearchDateComponents {
hour?: number;
}
/**
* A massaged version of {@link CGroup} suitable for being shown in search
* results.
*/
export interface SearchPerson {
id: string;
name?: string;
files: number[];
displayFaceID: string;
displayFaceFile: EnteFile;
}
/**
* A city as identified by a static dataset.
*
@@ -105,72 +128,3 @@ export type City = Location & {
/** Name of the city. */
name: string;
};
// TODO-cgroup: Audit below
export enum SuggestionType {
DATE = "DATE",
LOCATION = "LOCATION",
COLLECTION = "COLLECTION",
FILE_NAME = "FILE_NAME",
PERSON = "PERSON",
FILE_CAPTION = "FILE_CAPTION",
FILE_TYPE = "FILE_TYPE",
CLIP = "CLIP",
CITY = "CITY",
}
export interface Suggestion {
type: SuggestionType;
label: string;
value:
| SearchDateComponents
| number[]
| SearchPerson
| LocationTag
| City
| FileType
| ClipSearchScores;
}
export interface SearchQuery {
suggestion?: SearchSuggestion;
date?: SearchDateComponents;
location?: LocationTag;
city?: City;
collection?: number;
files?: number[];
person?: SearchPerson;
fileType?: FileType;
clip?: ClipSearchScores;
}
export interface SearchResultSummary {
optionName: string;
fileCount: number;
}
export type SearchSuggestion = { label: string } & (
| { type: "collection"; collectionID: number }
| { type: "files"; fileIDs: number[] }
| { type: "fileType"; fileType: FileType }
| { type: "date"; dateComponents: SearchDateComponents }
| { type: "location"; locationTag: LocationTag }
| { type: "city"; city: City }
| { type: "clip"; clipScoreForFileID: Map<number, number> }
| { type: "cgroup"; cgroup: SearchPerson }
);
/**
* An option shown in the the search bar's select dropdown.
*
* The option includes essential data that is necessary to show a corresponding
* entry in the dropdown. If the user selects the option, then we will re-run
* the search, using the data to filter the list of files shown to the user.
*/
export interface SearchOption extends Suggestion {
fileCount: number;
previewFiles: EnteFile[];
}
export type ClipSearchScores = Map<number, number>;

28
web/packages/new/photos/services/search/ui.ts
View File

@@ -1,28 +0,0 @@
import { t } from "i18next";
import { SuggestionType } from "./types";
/**
* Return a localized label for the given suggestion {@link type}.
*/
export const labelForSuggestionType = (type: SuggestionType) => {
switch (type) {
case SuggestionType.DATE:
return t("date");
case SuggestionType.LOCATION:
return t("location");
case SuggestionType.CITY:
return t("location");
case SuggestionType.COLLECTION:
return t("album");
case SuggestionType.FILE_NAME:
return t("file_name");
case SuggestionType.PERSON:
return t("person");
case SuggestionType.FILE_CAPTION:
return t("description");
case SuggestionType.FILE_TYPE:
return t("file_type");
case SuggestionType.CLIP:
return t("magic");
}
};

258
web/packages/new/photos/services/search/worker.ts
View File

@@ -3,6 +3,7 @@ import type { Location } from "@/base/types";
import type { Collection } from "@/media/collection";
import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata";
import type { EnteFile } from "@/new/photos/types/file";
import { ensure } from "@/utils/ensure";
import { nullToUndefined } from "@/utils/transform";
import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata";
import type { Component } from "chrono-node";
@@ -16,16 +17,14 @@ import {
} from "../user-entity";
import type {
City,
DateSearchResult,
LabelledFileType,
LabelledSearchDateComponents,
LocalizedSearchData,
Searchable,
SearchableData,
SearchDateComponents,
SearchQuery,
Suggestion,
SearchSuggestion,
} from "./types";
import { SuggestionType } from "./types";
/**
* A web worker that runs the search asynchronously so that the main thread
@@ -89,10 +88,18 @@ export class SearchWorker {
/**
* Return {@link EnteFile}s that satisfy the given {@link suggestion}.
*/
filterSearchableFiles(suggestion: SearchQuery) {
return this.searchableData.files.filter((f) =>
isMatchingFile(f, suggestion),
);
filterSearchableFiles(suggestion: SearchSuggestion) {
return filterSearchableFiles(this.searchableData.files, suggestion);
}
/**
* Batched variant of {@link filterSearchableFiles}.
*/
filterSearchableFilesMulti(suggestions: SearchSuggestion[]) {
const files = this.searchableData.files;
return suggestions
.map((sg) => [filterSearchableFiles(files, sg), sg] as const)
.filter(([files]) => files.length);
}
}
@@ -109,59 +116,83 @@ const suggestionsForString = (
{ locale, holidays, labelledFileTypes }: LocalizedSearchData,
locationTags: Searchable<LocationTag>[],
cities: Searchable<City>[],
): Suggestion[] =>
): SearchSuggestion[] =>
[
// <-- caption suggestions will be inserted here by our caller.
fileTypeSuggestions(s, labelledFileTypes),
dateSuggestions(s, locale, holidays),
locationSuggestions(s, locationTags, cities),
collectionSuggestions(s, collections),
suggestionForFiles(fileNameMatches(s, files), searchString),
suggestionForFiles(fileCaptionMatches(s, files), searchString),
fileNameSuggestion(s, searchString, files),
fileCaptionSuggestion(s, searchString, files),
].flat();
const collectionSuggestions = (s: string, collections: Collection[]) =>
const collectionSuggestions = (
s: string,
collections: Collection[],
): SearchSuggestion[] =>
collections
.filter(({ name }) => name.toLowerCase().includes(s))
.map(({ id, name }) => ({
type: SuggestionType.COLLECTION,
value: id,
type: "collection",
collectionID: id,
label: name,
}));
const fileNameMatches = (s: string, files: EnteFile[]) => {
const fileTypeSuggestions = (
s: string,
labelledFileTypes: Searchable<LabelledFileType>[],
): SearchSuggestion[] =>
labelledFileTypes
.filter(({ lowercasedName }) => lowercasedName.startsWith(s))
.map(({ fileType, label }) => ({ type: "fileType", fileType, label }));
const fileNameSuggestion = (
s: string,
searchString: string,
files: EnteFile[],
): SearchSuggestion[] => {
// Convert the search string to a number. This allows searching a file by
// its exact (integral) ID.
const sn = Number(s) || undefined;
return files.filter(
({ id, metadata }) =>
id === sn || metadata.title.toLowerCase().includes(s),
);
const fileIDs = files
.filter(
({ id, metadata }) =>
id === sn || metadata.title.toLowerCase().includes(s),
)
.map((f) => f.id);
return fileIDs.length
? [{ type: "fileName", fileIDs, label: searchString }]
: [];
};
const suggestionForFiles = (matchingFiles: EnteFile[], searchString: string) =>
matchingFiles.length
? {
type: SuggestionType.FILE_NAME,
value: matchingFiles.map((f) => f.id),
label: searchString,
}
: [];
const fileCaptionSuggestion = (
s: string,
searchString: string,
files: EnteFile[],
): SearchSuggestion[] => {
const fileIDs = files
.filter((file) =>
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
file.pubMagicMetadata?.data?.caption?.toLowerCase().includes(s),
)
.map((f) => f.id);
const fileCaptionMatches = (s: string, files: EnteFile[]) =>
files.filter((file) =>
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
file.pubMagicMetadata?.data?.caption?.toLowerCase().includes(s),
);
return fileIDs.length
? [{ type: "fileCaption", fileIDs, label: searchString }]
: [];
};
const dateSuggestions = (
s: string,
locale: string,
holidays: Searchable<DateSearchResult>[],
) =>
holidays: Searchable<LabelledSearchDateComponents>[],
): SearchSuggestion[] =>
parseDateComponents(s, locale, holidays).map(({ components, label }) => ({
type: SuggestionType.DATE,
value: components,
type: "date",
dateComponents: components,
label,
}));
@@ -182,15 +213,18 @@ const dateSuggestions = (
const parseDateComponents = (
s: string,
locale: string,
holidays: Searchable<DateSearchResult>[],
): DateSearchResult[] =>
holidays: Searchable<LabelledSearchDateComponents>[],
): LabelledSearchDateComponents[] =>
[
parseChrono(s, locale),
parseYearComponents(s),
holidays.filter(searchableIncludes(s)),
].flat();
const parseChrono = (s: string, locale: string): DateSearchResult[] =>
const parseChrono = (
s: string,
locale: string,
): LabelledSearchDateComponents[] =>
chrono
.parse(s)
.map((result) => {
@@ -224,7 +258,7 @@ const parseChrono = (s: string, locale: string): DateSearchResult[] =>
.filter((x) => x !== undefined);
/** chrono does not parse years like "2024", so do it manually. */
const parseYearComponents = (s: string): DateSearchResult[] => {
const parseYearComponents = (s: string): LabelledSearchDateComponents[] => {
// s is already trimmed.
if (s.length == 4) {
const year = parseInt(s);
@@ -271,7 +305,7 @@ const locationSuggestions = (
s: string,
locationTags: Searchable<LocationTag>[],
cities: Searchable<City>[],
) => {
): SearchSuggestion[] => {
const matchingLocationTags = locationTags.filter(searchableIncludes(s));
const matchingLocationTagLNames = new Set(
@@ -285,77 +319,77 @@ const locationSuggestions = (
);
return [
matchingLocationTags.map((t) => ({
type: SuggestionType.LOCATION,
value: t,
label: t.name,
})),
matchingCities.map((c) => ({
type: SuggestionType.CITY,
value: c,
label: c.name,
})),
matchingLocationTags.map(
(locationTag): SearchSuggestion => ({
type: "location",
locationTag,
label: locationTag.name,
}),
),
matchingCities.map(
(city): SearchSuggestion => ({
type: "city",
city,
label: city.name,
}),
),
].flat();
};
const fileTypeSuggestions = (
s: string,
labelledFileTypes: Searchable<LabelledFileType>[],
const filterSearchableFiles = (
files: EnteFile[],
suggestion: SearchSuggestion,
) =>
labelledFileTypes
.filter(searchableIncludes(s))
.map(({ fileType, label }) => ({
label,
value: fileType,
type: SuggestionType.FILE_TYPE,
}));
sortMatchesIfNeeded(
files.filter((f) => isMatchingFile(f, suggestion)),
suggestion,
);
/**
* Return true if file satisfies the given {@link query}.
*/
const isMatchingFile = (file: EnteFile, query: SearchQuery) => {
if (query.collection) {
return query.collection === file.collectionID;
const isMatchingFile = (file: EnteFile, suggestion: SearchSuggestion) => {
switch (suggestion.type) {
case "collection":
return suggestion.collectionID === file.collectionID;
case "fileType":
return suggestion.fileType === file.metadata.fileType;
case "fileName":
return suggestion.fileIDs.includes(file.id);
case "fileCaption":
return suggestion.fileIDs.includes(file.id);
case "date":
return isDateComponentsMatch(
suggestion.dateComponents,
fileCreationPhotoDate(file, getPublicMagicMetadataSync(file)),
);
case "location": {
const location = fileLocation(file);
if (!location) return false;
return isInsideLocationTag(location, suggestion.locationTag);
}
case "city": {
const location = fileLocation(file);
if (!location) return false;
return isInsideCity(location, suggestion.city);
}
case "clip":
return suggestion.clipScoreForFileID.has(file.id);
case "person":
// return query.person.files.includes(file.id);
// TODO-Cluster implement me
return false;
}
if (query.date) {
return isDateComponentsMatch(
query.date,
fileCreationPhotoDate(file, getPublicMagicMetadataSync(file)),
);
}
if (query.location) {
const location = fileLocation(file);
if (!location) return false;
return isInsideLocationTag(location, query.location);
}
if (query.city) {
const location = fileLocation(file);
if (!location) return false;
return isInsideCity(location, query.city);
}
if (query.files) {
return query.files.includes(file.id);
}
if (query.person) {
return query.person.files.includes(file.id);
}
if (typeof query.fileType !== "undefined") {
return query.fileType === file.metadata.fileType;
}
if (typeof query.clip !== "undefined") {
return query.clip.has(file.id);
}
return false;
};
const isDateComponentsMatch = (
@@ -412,3 +446,21 @@ const isWithinRadius = (
* major axis (a) has to be scaled by the secant of the latitude.
*/
const radiusScaleFactor = (lat: number) => 1 / Math.cos(lat * (Math.PI / 180));
/**
* Sort the files if necessary.
*
* Currently, only the CLIP results are sorted (by their score), in the other
* cases the files are displayed chronologically (when displaying them in search
* results) or arbitrarily (when showing them in the search option preview).
*/
const sortMatchesIfNeeded = (
files: EnteFile[],
suggestion: SearchSuggestion,
) => {
if (suggestion.type != "clip") return files;
// Sort CLIP matches by their corresponding scores.
const score = ({ id }: EnteFile) =>
ensure(suggestion.clipScoreForFileID.get(id));
return files.sort((a, b) => score(b) - score(a));
};

20
web/packages/shared/components/CodeBlock/index.tsx
View File

@@ -1,20 +1,14 @@
import { FreeFlowText } from "@ente/shared/components/Container";
import EnteSpinner from "@ente/shared/components/EnteSpinner";
import type { BoxProps } from "@mui/material";
import { type BoxProps, styled } from "@mui/material";
import React from "react";
import CopyButton from "./CopyButton";
import { CodeWrapper, CopyButtonWrapper, Wrapper } from "./styledComponents";
type Iprops = React.PropsWithChildren<{
code: string | null;
wordBreak?: "normal" | "break-all" | "keep-all" | "break-word";
}>;
export default function CodeBlock({
code,
wordBreak,
...props
}: BoxProps<"div", Iprops>) {
export default function CodeBlock({ code, ...props }: BoxProps<"div", Iprops>) {
if (!code) {
return (
<Wrapper>
@@ -25,9 +19,7 @@ export default function CodeBlock({
return (
<Wrapper {...props}>
<CodeWrapper>
<FreeFlowText style={{ wordBreak: wordBreak }}>
{code}
</FreeFlowText>
<FreeFlowText>{code}</FreeFlowText>
</CodeWrapper>
<CopyButtonWrapper>
<CopyButton code={code} />
@@ -35,3 +27,9 @@ export default function CodeBlock({
</Wrapper>
);
}
const FreeFlowText = styled("div")`
word-break: break-word;
min-width: 30%;
text-align: left;
`;

6
web/packages/shared/components/Container.tsx
View File

@@ -31,12 +31,6 @@ export const FlexWrapper = styled(Box)`
align-items: center;
`;
export const FreeFlowText = styled("div")`
word-break: break-word;
min-width: 30%;
text-align: left;
`;
export const SpaceBetweenFlex = styled(FlexWrapper)`
justify-content: space-between;
`;