TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[desktop] Improve first CLIP query behaviour (#6369)

Browse Source 
Keep some leeway so that the first CLIP query in a session also gets
resolved (on app starts after the model has been downloaded).
This commit is contained in:
Manav Rathi
2025-06-26 08:23:20 +05:30
committed by GitHub
parent 1babaf529e b981152dee
commit a16b20122d
13 changed files with 50 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
desktop/src/main/services/ml-worker.ts
View File

@@ -184,14 +184,13 @@ const downloadModel = async (saveLocation: string, name: string) => {
/**
* Create an ONNX {@link InferenceSession} with some defaults.
*/
const createInferenceSession = async (modelPath: string) => {
return await ort.InferenceSession.create(modelPath, {
const createInferenceSession = async (modelPath: string) =>
ort.InferenceSession.create(modelPath, {
// Restrict the number of threads to 1.
intraOpNumThreads: 1,
// Be more conservative with RAM usage.
enableCpuMemArena: false,
});
};
const cachedCLIPImageSession = makeCachedInferenceSession(
"mobileclip_s2_image_opset18_rgba_opt.onnx",
@@ -233,9 +232,11 @@ const getTokenizer = () => (_tokenizer ??= new Tokenizer());
export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {
const sessionOrSkip = await Promise.race([
cachedCLIPTextSession(),
// Wait for a tick to get the session promise to resolved the first time
// this code runs on each app start (and the model has been downloaded).
wait(0).then(() => 1),
// Wait a bit to get the session promise to resolved the first time this
// code runs on each app start (in these cases the model will already be
// downloaded, so session creation should take only a 1 or 2 ticks: file
// system stat, and ort.InferenceSession.create).
wait(50).then(() => 1),
]);
// Don't wait for the download to complete.

9
web/apps/photos/src/components/FileList.tsx
View File

@@ -123,6 +123,10 @@ export interface FileListProps {
*/
modePlus?: GalleryBarMode | "search";
showAppDownloadBanner?: boolean;
/**
* If `true`, then the current listing is showing magic search results.
*/
isMagicSearchResult?: boolean;
selectable?: boolean;
setSelected: (
selected: SelectedState | ((selected: SelectedState) => SelectedState),
@@ -167,6 +171,7 @@ export const FileList: React.FC<FileListProps> = ({
modePlus,
annotatedFiles,
showAppDownloadBanner,
isMagicSearchResult,
selectable,
selected,
setSelected,
@@ -235,7 +240,7 @@ export const FileList: React.FC<FileListProps> = ({
),
);
}
if (galleryContext.isClipSearchResult) {
if (isMagicSearchResult) {
noGrouping(timeStampList);
} else {
groupByTime(timeStampList);
@@ -277,7 +282,7 @@ export const FileList: React.FC<FileListProps> = ({
annotatedFiles,
galleryContext.photoListHeader,
publicCollectionGalleryContext.photoListHeader,
galleryContext.isClipSearchResult,
isMagicSearchResult,
]);
useEffect(() => {

3
web/apps/photos/src/components/FileListWithViewer.tsx
View File

@@ -53,6 +53,7 @@ export type FileListWithViewerProps = {
| "mode"
| "modePlus"
| "showAppDownloadBanner"
| "isMagicSearchResult"
| "selectable"
| "selected"
| "setSelected"
@@ -89,6 +90,7 @@ export const FileListWithViewer: React.FC<FileListWithViewerProps> = ({
files,
enableDownload,
showAppDownloadBanner,
isMagicSearchResult,
selectable,
selected,
setSelected,
@@ -175,6 +177,7 @@ export const FileListWithViewer: React.FC<FileListWithViewerProps> = ({
mode,
modePlus,
showAppDownloadBanner,
isMagicSearchResult,
selectable,
selected,
setSelected,

4
web/apps/photos/src/components/Sidebar.tsx
View File

@@ -102,8 +102,8 @@ import {
isSubscriptionPastDue,
isSubscriptionStripe,
leaveFamily,
pullUserDetails,
redirectToCustomerPortal,
syncUserDetails,
userDetailsAddOnBonuses,
type UserDetails,
} from "ente-new/photos/services/user-details";
@@ -230,7 +230,7 @@ const UserDetailsSection: React.FC<UserDetailsSectionProps> = ({
} = useModalVisibility();
useEffect(() => {
if (sidebarOpen) void syncUserDetails();
if (sidebarOpen) void pullUserDetails();
}, [sidebarOpen]);
const isNonAdminFamilyMember = useMemo(

15
web/apps/photos/src/pages/gallery.tsx
View File

@@ -91,7 +91,7 @@ import {
import type { SearchOption } from "ente-new/photos/services/search/types";
import { initSettings } from "ente-new/photos/services/settings";
import {
initUserDetailsOrTriggerSync,
initUserDetailsOrTriggerPull,
redirectToCustomerPortal,
userDetailsSnapshot,
verifyStripeSubscription,
@@ -137,7 +137,6 @@ const defaultGalleryContext: GalleryContextType = {
userIDToEmailMap: null,
emailList: null,
openHiddenSection: () => null,
isClipSearchResult: null,
selectedFile: null,
setSelectedFiles: () => null,
};
@@ -202,9 +201,6 @@ const Page: React.FC = () => {
const peopleState = usePeopleStateSnapshot();
const [isClipSearchResult, setIsClipSearchResult] =
useState<boolean>(false);
// The (non-sticky) header shown at the top of the gallery items.
const [photoListHeader, setPhotoListHeader] =
useState<TimeStampListItem>(null);
@@ -296,7 +292,7 @@ const Page: React.FC = () => {
return;
}
initSettings();
await initUserDetailsOrTriggerSync();
await initUserDetailsOrTriggerPull();
setupSelectAllKeyBoardShortcutHandler();
dispatch({ type: "showAll" });
setIsFirstLoad(isFirstLogin());
@@ -758,7 +754,6 @@ const Page: React.FC = () => {
} else {
dispatch({ type: "exitSearch" });
}
setIsClipSearchResult(type == "clip");
};
const openUploader = (intent?: UploadTypeSelectorIntent) => {
@@ -821,7 +816,7 @@ const Page: React.FC = () => {
// 2. Construct a fake a metadata object with the updates
// reflected in it.
//
// 3. The caller (eventually) triggers a remote sync in the
// 3. The caller (eventually) triggers a remote pull in the
// background, but meanwhile uses this updated metadata.
//
// TODO(RE): Replace with file fetch?
@@ -895,7 +890,6 @@ const Page: React.FC = () => {
userIDToEmailMap: state.emailByUserID,
emailList: state.shareSuggestionEmails,
openHiddenSection,
isClipSearchResult,
selectedFile: selected,
setSelectedFiles: setSelected,
}}
@@ -1094,6 +1088,9 @@ const Page: React.FC = () => {
showAppDownloadBanner={
state.collectionFiles.length < 30 && !isInSearchMode
}
isMagicSearchResult={
state.searchSuggestion?.type == "clip"
}
selectable={true}
selected={selected}
setSelected={setSelected}

2
web/apps/photos/src/pages/shared-albums.tsx
View File

@@ -362,7 +362,7 @@ export default function PublicCollectionGallery() {
setPublicCollection(null);
setPublicFiles(null);
} else {
log.error("failed to sync public album with remote", e);
log.error("Public album remote pull failed", e);
}
} finally {
hideLoadingBar();

1
web/apps/photos/src/types/gallery/index.ts
View File

@@ -46,7 +46,6 @@ export interface GalleryContextType {
userIDToEmailMap: Map<number, string>;
emailList: string[];
openHiddenSection: (callback?: () => void) => void;
isClipSearchResult: boolean;
setSelectedFiles: (value) => void;
selectedFile: SelectedState;
}

4
web/apps/photos/tests/upload.test.ts
View File

@@ -18,7 +18,7 @@ import {
savedCollectionFiles,
savedCollections,
} from "ente-new/photos/services/photos-fdb";
import { getUserDetailsV2 } from "ente-new/photos/services/user-details";
import { userDetailsSnapshot } from "ente-new/photos/services/user-details";
const DATE_TIME_PARSING_TEST_FILE_NAMES = [
{
@@ -170,7 +170,7 @@ export async function testUpload() {
}
async function totalFileCountCheck(expectedState) {
const userDetails = await getUserDetailsV2();
const userDetails = userDetailsSnapshot();
if (expectedState.total_file_count === userDetails.fileCount) {
console.log("file count check passed ✅");
} else {

4
web/packages/gallery/services/file-data.ts
View File

@@ -196,8 +196,8 @@ export interface UpdatedFileDataFileIDsPage {
fileIDs: Set<number>;
/**
* The latest updatedAt (epoch microseconds) time obtained from remote in
* this batch of sync (from amongst all of the files in the batch, not just
* those that were filtered to be part of {@link fileIDs}).
* this batch being fetched (from amongst all of the files in the batch, not
* just those that were filtered to be part of {@link fileIDs}).
*/
lastUpdatedAt: number;
}

14
web/packages/new/photos/components/gallery/reducer.ts
View File

@@ -163,7 +163,7 @@ export interface GalleryState {
* Unsynced modifications are those whose effects have already been made on
* remote (so thus they have been "saved", so to say), but we still haven't
* yet refreshed our local state to incorporate them. The refresh will
* happen on the next "file sync", until then they remain as in-memory state
* happen on the next files pull, until then they remain as in-memory state
* in the reducer.
*/
collectionFiles: EnteFile[];
@@ -318,9 +318,9 @@ export interface GalleryState {
* the user's favorites, false otherwise) which should be used for that file
* instead of what we get from our local DB.
*
* The next time a sync with remote completes, we clear this map since
* thereafter just deriving {@link favoriteFileIDs} from our local files
* would reflect the correct state on remote too.
* The next time a remote pull completes, we clear this map since thereafter
* just deriving {@link favoriteFileIDs} from our local files would reflect
* the correct state on remote too.
*/
unsyncedFavoriteUpdates: Map<number, boolean>;
/**
@@ -338,9 +338,9 @@ export interface GalleryState {
* Each entry from a file ID to the magic metadata that should be used for
* that file instead of what we get from our local DB.
*
* The next time a sync with remote completes, we clear this map since
* thereafter the synced files themselves will reflect the latest private
* magic metadata.
* The next time a remote pull completes, we clear this map since thereafter
* the synced files themselves will reflect the latest private magic
* metadata.
*/
unsyncedPrivateMagicMetadataUpdates: Map<
number,

2
web/packages/new/photos/services/ml/index.ts
View File

@@ -56,7 +56,7 @@ class MLState {
*
* - On app start, this is read from local storage during {@link initML}.
*
* - It gets updated when we sync with remote (so if the user
* - It gets updated when we pull from remote (so if the user
* enables/disables ML on a different device, this local value will also
* become true/false).
*

2
web/packages/new/photos/services/pull.ts
View File

@@ -79,7 +79,7 @@ interface PullFilesOpts {
* This is a subset of a full remote pull, independently exposed for use at
* times when we only want to pull the file related information (e.g. we just
* made some API request that modified collections or files, and so now want to
* sync our local changes to match remote).
* update our local changes to match remote).
*
* See also: [Note: Remote pull]
*

21
web/packages/new/photos/services/user-details.ts
View File

@@ -183,12 +183,12 @@ export const logoutUserDetails = () => {
*
* This assumes that the user is already logged in.
*/
export const initUserDetailsOrTriggerSync = async () => {
export const initUserDetailsOrTriggerPull = async () => {
const saved = await getKV("userDetails");
if (saved) {
setUserDetailsSnapshot(UserDetails.parse(saved));
} else {
void syncUserDetails();
void pullUserDetails();
}
};
@@ -231,8 +231,8 @@ const setUserDetailsSnapshot = (snapshot: UserDetails) => {
* Fetch the user's details from remote and save them in local storage for
* subsequent lookup, and also update our in-memory snapshots.
*/
export const syncUserDetails = async () => {
const userDetails = await getUserDetailsV2();
export const pullUserDetails = async () => {
const userDetails = await getUserDetails();
await setKV("userDetails", userDetails);
setUserDetailsSnapshot(userDetails);
@@ -248,7 +248,7 @@ export const syncUserDetails = async () => {
/**
* Fetch the user details for the currently logged in user from remote.
*/
export const getUserDetailsV2 = async () => {
const getUserDetails = async () => {
const res = await fetch(await apiURL("/users/details/v2"), {
headers: await authenticatedRequestHeaders(),
});
@@ -326,7 +326,7 @@ export const verifyStripeSubscription = async (
}),
}),
);
await syncUserDetails();
await pullUserDetails();
return userDetailsSnapshot()!.subscription;
};
@@ -342,7 +342,7 @@ export const activateStripeSubscription = async () => {
headers: await authenticatedRequestHeaders(),
}),
);
return syncUserDetails();
return pullUserDetails();
};
/**
@@ -356,7 +356,7 @@ export const cancelStripeSubscription = async () => {
headers: await authenticatedRequestHeaders(),
}),
);
return syncUserDetails();
return pullUserDetails();
};
const paymentsAppOrigin = "https://payments.ente.io";
@@ -539,7 +539,8 @@ const getFamiliesTokenAndURL = async () => {
/**
* Update remote to indicate that the user wants to leave the family plan that
* they are part of, then our local sync user details with remote.
* they are part of, then update our local user details by pulling the latest
* ones from remote.
*/
export const leaveFamily = async () => {
ensureOk(
@@ -548,7 +549,7 @@ export const leaveFamily = async () => {
headers: await authenticatedRequestHeaders(),
}),
);
return syncUserDetails();
return pullUserDetails();
};
/**