From e35b4eac407c429df48d5e82d4125e40c20f36be Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 11:29:27 +0530 Subject: [PATCH 01/12] Refill queue even after first refill --- web/packages/gallery/services/video.ts | 27 ++++++++------------------ 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index d038942598..0594596e48 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -675,27 +675,16 @@ const processQueue = async () => { while (isVideoProcessingEnabled()) { let item = _state.liveQueue.shift(); if (!item) { - if (!bq && _state.haveSyncedOnce) { - /* initialize */ - bq = await backfillQueue(userID); - } - if (bq) { - switch (bq.length) { - case 0: - /* no more items to backfill */ - break; - case 1 /* last item. take it, and refill queue */: - item = bq.pop(); - bq = await backfillQueue(userID); - break; - default: - /* more than one item. take it */ - item = bq.pop(); - break; + // Initialize or refill queue. + if (!bq?.length) { + if (_state.haveSyncedOnce) { + bq = await backfillQueue(userID); + } else { + log.info("Not attempting backfill until first sync"); } - } else { - log.info("Not backfilling since we haven't synced yet"); } + // Take item if queue is not empty. + if (bq?.length) item = bq.pop(); } if (item) { try { From 08346e5bcdecc1770f9b3f03c144cf9c5b716bbd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 11:36:00 +0530 Subject: [PATCH 02/12] it was already revoking --- web/packages/base/utils/web.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/web/packages/base/utils/web.ts b/web/packages/base/utils/web.ts index f51583ae11..cb45280990 100644 --- a/web/packages/base/utils/web.ts +++ b/web/packages/base/utils/web.ts @@ -3,11 +3,12 @@ * folder by appending a temporary element to the DOM. * * @param url The URL that we want to download. See also - * {@link downloadAndRevokeObjectURL} and {@link downloadString}. + * {@link downloadAndRevokeObjectURL} and {@link downloadString}. The URL is + * revoked after initiating the download. * * @param fileName The name of downloaded file. */ -export const downloadURL = (url: string, fileName: string) => { +export const downloadAndRevokeObjectURL = (url: string, fileName: string) => { const a = document.createElement("a"); a.style.display = "none"; a.href = url; @@ -18,15 +19,6 @@ export const downloadURL = (url: string, fileName: string) => { a.remove(); }; -/** - * A variant of {@link downloadURL} that also revokes the provided - * {@link objectURL} after initiating the download. - */ -export const downloadAndRevokeObjectURL = (url: string, fileName: string) => { - downloadURL(url, fileName); - URL.revokeObjectURL(url); -}; - /** * Save the given string {@link s} as a file in the user's download folder. * From 442d6526bede9f8e0d239faa1eacfce2334f0e0b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 11:45:40 +0530 Subject: [PATCH 03/12] sopt color something (I'm not sure what, but I think react-select itself) overrides the color for the option's root element to white when displaying the search bar on the search results screen itself. as a workaround, provide a explicit color to the text. steps to reproduce (light mode): - search for something (all options look normal) - select an option - search for something on this results screen itself - note how the search options titles are in white --- web/packages/new/photos/components/SearchBar.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/packages/new/photos/components/SearchBar.tsx b/web/packages/new/photos/components/SearchBar.tsx index 0240bbdf4c..db1297e3f2 100644 --- a/web/packages/new/photos/components/SearchBar.tsx +++ b/web/packages/new/photos/components/SearchBar.tsx @@ -473,7 +473,11 @@ const OptionContents = ({ data: option }: { data: SearchOption }) => ( > {option.suggestion.label} From be2665f57f9a31cdb5bb5e262b0a0f0e1ee1fb47 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 12:29:17 +0530 Subject: [PATCH 04/12] Set sv = 1 for files that are skipped --- desktop/src/main/services/ffmpeg-worker.ts | 2 ++ web/packages/gallery/services/video.ts | 13 ++++++++++- web/packages/media/file-metadata.ts | 27 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/desktop/src/main/services/ffmpeg-worker.ts b/desktop/src/main/services/ffmpeg-worker.ts index c186f0a32b..86604c1f63 100644 --- a/desktop/src/main/services/ffmpeg-worker.ts +++ b/desktop/src/main/services/ffmpeg-worker.ts @@ -227,6 +227,8 @@ const ffmpegGenerateHLSPlaylistAndSegments = async ( // going to use for the conversion), then a streaming variant is not much // use. Skip such cases. // + // See also: [Note: Marking files which do not need video processing] + // // --- // // [Note: HEVC/H.265 issues] diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index 0594596e48..79cf1d5090 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -12,6 +12,10 @@ import { getKV, getKVN, setKV } from "ente-base/kv"; import { ensureLocalUser } from "ente-base/local-user"; import log from "ente-base/log"; import { fileLogID, type EnteFile } from "ente-media/file"; +import { + filePublicMagicMetadata, + updateRemotePublicMagicMetadata, +} from "ente-media/file-metadata"; import { FileType } from "ente-media/file-type"; import { getAllLocalFiles, @@ -732,9 +736,14 @@ const backfillQueue = async ( const videoFiles = uniqueFilesByID( allCollectionFiles.filter( (f) => + // Only files the user owns. f.ownerID == userID && + // Only videos. f.metadata.fileType == FileType.video && - !localTrashFileIDs.has(f.id), + // Not in trash. + !localTrashFileIDs.has(f.id) && + // See: [Note: Marking files which do not need video processing] + filePublicMagicMetadata(f)?.sv != 1, ), ); @@ -845,6 +854,8 @@ const processQueueItem = async ({ if (!res) { log.info(`Generate HLS for ${fileLogID(file)} | not-required`); + // See: [Note: Marking files which do not need video processing] + await updateRemotePublicMagicMetadata(file, { sv: 1 }); return; } diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 4fcda2e238..e59d8d88e5 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -274,6 +274,33 @@ export interface PublicMagicMetadata { */ caption?: string; uploaderName?: string; + /** + * An arbitrary integer set to indicate that this file should be skipped for + * the purpose of HLS generation. + * + * Current semantics: + * + * - if 1, skip this file + * - otherwise attempt processing + * + * [Note: Marking files which do not need video processing] + * + * Some video files do not require generation of a HLS stream. The current + * logic is H.264 files less than 10 MB, but this might change in future + * clients. + * + * For such skipped files, there thus won't be a HLS playlist generated. + * However, we still need a way to indicate to other clients that this file + * has already been looked at. + * + * To that end, we add a flag to the public magic metadata for the file. To + * allow future flexibility, this flag is an integer "streaming version". + * Currently it is set to 1 by a client who recognizes that this file does + * not need processing, and other clients can ignore this file if they find + * sv == 1. In the future, there might be other values for sv (e.g. if the + * skip logic changes). + */ + sv?: number; } /** From 0284287c9c397c2ddacdd4f6b972a36bdce6ed72 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 13:14:14 +0530 Subject: [PATCH 05/12] toggle --- web/apps/photos/src/components/Sidebar.tsx | 20 ++++ web/packages/gallery/services/video.ts | 97 ++++++++++++++++++- .../photos/components/utils/use-snapshot.ts | 14 +++ 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar.tsx b/web/apps/photos/src/components/Sidebar.tsx index d79afb1942..cbf66df53a 100644 --- a/web/apps/photos/src/components/Sidebar.tsx +++ b/web/apps/photos/src/components/Sidebar.tsx @@ -61,6 +61,10 @@ import log from "ente-base/log"; import { savedLogs } from "ente-base/log-web"; import { customAPIHost } from "ente-base/origins"; import { downloadString } from "ente-base/utils/web"; +import { + isHLSGenerationSupported, + toggleHLSGeneration, +} from "ente-gallery/services/video"; import { DeleteAccount } from "ente-new/photos/components/DeleteAccount"; import { DropdownInput } from "ente-new/photos/components/DropdownInput"; import { MLSettings } from "ente-new/photos/components/sidebar/MLSettings"; @@ -71,6 +75,7 @@ import { } from "ente-new/photos/components/utils/dialog"; import { downloadAppDialogAttributes } from "ente-new/photos/components/utils/download"; import { + useHLSGenerationStatusSnapshot, useSettingsSnapshot, useUserDetailsSnapshot, } from "ente-new/photos/components/utils/use-snapshot"; @@ -769,6 +774,9 @@ const Preferences: React.FC = ({ const { show: showMLSettings, props: mlSettingsVisibilityProps } = useModalVisibility(); + const hlsGenStatusSnapshot = useHLSGenerationStatusSnapshot(); + const isHLSGenerationEnabled = !!hlsGenStatusSnapshot?.enabled; + useEffect(() => { if (open) void syncSettings(); }, [open]); @@ -799,6 +807,18 @@ const Preferences: React.FC = ({ /> )} + {isHLSGenerationSupported() && ( + // TODO(HLS): Visual look + + + + + + )} } label={t("map")} diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index 79cf1d5090..43c55a9f2f 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -49,6 +49,10 @@ import { type TimestampedFileSystemUploadItem, } from "./upload"; +export type HLSGenerationStatus = + | { enabled: false } + | { enabled: true; estimatedPendingCount?: number }; + interface VideoProcessingQueueItem { /** * The {@link EnteFile} (guaranteed to be of {@link FileType.video}) whose @@ -75,6 +79,20 @@ const idleWaitMax = idleWaitInitial * 2 ** 6; /* 640 sec */ * This entire object will be reset on logout. */ class VideoState { + /** + * `true` if the generation of HLS streams has been enabled on this client. + */ + isHLSGenerationEnabled = false; + /** + * Subscriptions to {@link HLSGenerationStatus} updates attached using + * {@link hlsGenerationStatusSubscribe}. + */ + hlsGenerationStatusListeners: (() => void)[] = []; + /** + * Snapshot of the {@link HLSGenerationStatus} returned by the + * {@link hlsGenerationStatusSnapshot} function. + */ + hlsGenerationStatusSnapshot: HLSGenerationStatus | undefined; /** * Queue of recently uploaded items waiting to be processed. */ @@ -129,6 +147,77 @@ export const resetVideoState = () => { _state = new VideoState(); }; +/** + * A function that can be used to subscribe to updates in the HLS generation + * settings and status. + * + * See: [Note: Snapshots and useSyncExternalStore]. + */ +export const hlsGenerationStatusSubscribe = ( + onChange: () => void, +): (() => void) => { + _state.hlsGenerationStatusListeners.push(onChange); + return () => { + _state.hlsGenerationStatusListeners = + _state.hlsGenerationStatusListeners.filter((l) => l != onChange); + }; +}; + +/** + * Return the last know, cached {@link HLSGenerationStatus}. + * + * See also {@link hlsGenerationStatusSubscribe}. + * + * A return value of `undefined` indicates that the HLS generation subsystem has + * not been initialized yet. + */ +export const hlsGenerationStatusSnapshot = (): + | HLSGenerationStatus + | undefined => _state.hlsGenerationStatusSnapshot; + +const setHLSGenerationStatusSnapshot = (snapshot: HLSGenerationStatus) => { + _state.hlsGenerationStatusSnapshot = snapshot; + _state.hlsGenerationStatusListeners.forEach((l) => l()); +}; + +/** + * Return `true` if this client is capable of generating HLS streams for the + * uploaded videos. + * + * This function implementation is written expecting to be called many times + * (e.g. during UI rendering). + */ +export const isHLSGenerationSupported = () => + // Keep this check fast, we get called many times. + isDesktop && + // TODO(HLS): + process.env.NEXT_PUBLIC_ENTE_WIP_VIDEO_STREAMING && + settingsSnapshot().isInternalUser; + +/** + * Enable or disable (toggle) the HLS generation on this client. + * + * When HLS generation is enabled, this client will process videos to generate a + * streamable variant of them. + * + * It can only be enabled when + */ +export const toggleHLSGeneration = () => { + if (!isHLSGenerationSupported()) { + assertionFailed(); + return; + } + + const enabled = !_state.isHLSGenerationEnabled; + + // Right now we only set the enabled setting. The estimated count status + // will get filled in when we tick. + setHLSGenerationStatusSnapshot({ enabled }); + + // Wake up the processor if needed. + if (enabled) tickNow(); +}; + export interface HLSPlaylistData { /** A data URL to a HLS playlist that streams the video. */ playlistURL: string; @@ -515,7 +604,7 @@ const syncProcessedFileIDs = async () => export const videoPrunePermanentlyDeletedFileIDsIfNeeded = async ( deletedFileIDs: Set, ) => { - if (!isDesktop) return; + if (!isHLSGenerationSupported()) return; const existing = await savedProcessedVideoFileIDs(); if (existing.size > 0) { @@ -544,7 +633,7 @@ export const videoPrunePermanentlyDeletedFileIDsIfNeeded = async ( * that have already been processed elsewhere. */ export const videoProcessingSyncIfNeeded = async () => { - if (!isDesktop) return; + if (!isHLSGenerationSupported()) return; if (!isVideoProcessingEnabled()) return; await syncProcessedFileIDs(); @@ -580,7 +669,7 @@ export const processVideoNewUpload = ( file: EnteFile, processableUploadItem: ProcessableUploadItem, ) => { - if (!isDesktop) return; + if (!isHLSGenerationSupported()) return; if (!isVideoProcessingEnabled()) return; if (file.metadata.fileType !== FileType.video) return; if (processableUploadItem instanceof File) { @@ -668,7 +757,7 @@ export const isVideoProcessingEnabled = () => * batches, and the externally triggered processing of live uploads. */ const processQueue = async () => { - if (!(isDesktop && isVideoProcessingEnabled())) { + if (!isHLSGenerationSupported() || !isVideoProcessingEnabled()) { assertionFailed(); /* we shouldn't have come here */ return; } diff --git a/web/packages/new/photos/components/utils/use-snapshot.ts b/web/packages/new/photos/components/utils/use-snapshot.ts index 3cf14122aa..21073d6155 100644 --- a/web/packages/new/photos/components/utils/use-snapshot.ts +++ b/web/packages/new/photos/components/utils/use-snapshot.ts @@ -1,3 +1,7 @@ +import { + hlsGenerationStatusSnapshot, + hlsGenerationStatusSubscribe, +} from "ente-gallery/services/video"; import { useSyncExternalStore } from "react"; import { mlStatusSnapshot, @@ -38,3 +42,13 @@ export const useMLStatusSnapshot = () => */ export const usePeopleStateSnapshot = () => useSyncExternalStore(peopleStateSubscribe, peopleStateSnapshot); + +/** + * A convenience hook that returns {@link hlsGenerationStatusSnapshot}, and also + * subscribes to updates. + */ +export const useHLSGenerationStatusSnapshot = () => + useSyncExternalStore( + hlsGenerationStatusSubscribe, + hlsGenerationStatusSnapshot, + ); From 116f22a85330c1949df334120d8e05c570e5c5d8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 13:33:00 +0530 Subject: [PATCH 06/12] vis --- web/apps/photos/src/components/Sidebar.tsx | 32 ++++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar.tsx b/web/apps/photos/src/components/Sidebar.tsx index cbf66df53a..95217b562d 100644 --- a/web/apps/photos/src/components/Sidebar.tsx +++ b/web/apps/photos/src/components/Sidebar.tsx @@ -7,6 +7,7 @@ import HealthAndSafetyIcon from "@mui/icons-material/HealthAndSafety"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; import LockOutlinedIcon from "@mui/icons-material/LockOutlined"; import NorthEastIcon from "@mui/icons-material/NorthEast"; +import ScienceIcon from "@mui/icons-material/Science"; import VisibilityOffIcon from "@mui/icons-material/VisibilityOff"; import { Box, @@ -33,6 +34,7 @@ import { RowButtonDivider, RowButtonGroup, RowButtonGroupHint, + RowButtonGroupTitle, RowSwitch, } from "ente-base/components/RowButton"; import { SpacedRow } from "ente-base/components/containers"; @@ -52,6 +54,7 @@ import { import { useBaseContext } from "ente-base/context"; import { getLocaleInUse, + pt, setLocaleInUse, supportedLocales, ut, @@ -807,18 +810,6 @@ const Preferences: React.FC = ({ /> )} - {isHLSGenerationSupported() && ( - // TODO(HLS): Visual look - - - - - - )} } label={t("map")} @@ -829,6 +820,23 @@ const Preferences: React.FC = ({ label={t("advanced")} onClick={showAdvancedSettings} /> + {isHLSGenerationSupported() && ( + // TODO(HLS): Visual look + + }> + {t("labs")} + + + + + + )} Date: Tue, 13 May 2025 14:14:16 +0530 Subject: [PATCH 07/12] init --- web/apps/photos/src/components/Sidebar.tsx | 2 +- web/apps/photos/src/pages/_app.tsx | 5 ++ web/packages/gallery/services/video.ts | 54 ++++++++++++++++++---- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar.tsx b/web/apps/photos/src/components/Sidebar.tsx index 95217b562d..dd51591057 100644 --- a/web/apps/photos/src/components/Sidebar.tsx +++ b/web/apps/photos/src/components/Sidebar.tsx @@ -832,7 +832,7 @@ const Preferences: React.FC = ({ /* TODO(HLS): */ pt("Streamable videos") } checked={isHLSGenerationEnabled} - onClick={toggleHLSGeneration} + onClick={() => void toggleHLSGeneration()} /> diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index f17ce008ee..6954ec5bc0 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -46,6 +46,10 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { resumeExportsIfNeeded } from "services/export"; import { photosLogout } from "services/logout"; +import { + initVideoProcessing, + isHLSGenerationSupportedTemp, +} from "ente-gallery/services/video"; import "photoswipe/dist/photoswipe.css"; import "styles/global.css"; import "styles/photoswipe.css"; @@ -108,6 +112,7 @@ const App: React.FC = ({ Component, pageProps }) => { }; if (isMLSupported) initML(); + if (isHLSGenerationSupportedTemp()) void initVideoProcessing(); electron.onOpenEnteURL(handleOpenEnteURL); electron.onAppUpdateAvailable(showUpdateDialog); diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index 43c55a9f2f..29300f79be 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -8,7 +8,7 @@ import { retryAsyncOperation, type PublicAlbumsCredentials, } from "ente-base/http"; -import { getKV, getKVN, setKV } from "ente-base/kv"; +import { getKV, getKVB, getKVN, setKV } from "ente-base/kv"; import { ensureLocalUser } from "ente-base/local-user"; import log from "ente-base/log"; import { fileLogID, type EnteFile } from "ente-media/file"; @@ -181,11 +181,11 @@ const setHLSGenerationStatusSnapshot = (snapshot: HLSGenerationStatus) => { }; /** - * Return `true` if this client is capable of generating HLS streams for the + * Return `true` if this client is capable of generating HLS streams for * uploaded videos. * - * This function implementation is written expecting to be called many times - * (e.g. during UI rendering). + * This function implementation is fast and can be called many times (e.g. + * during UI rendering). */ export const isHLSGenerationSupported = () => // Keep this check fast, we get called many times. @@ -194,15 +194,48 @@ export const isHLSGenerationSupported = () => process.env.NEXT_PUBLIC_ENTE_WIP_VIDEO_STREAMING && settingsSnapshot().isInternalUser; +// TODO(HLS): Only the isDesktop flag is needed eventually. +export const isHLSGenerationSupportedTemp = () => + isDesktop && + // TODO(HLS): + process.env.NEXT_PUBLIC_ENTE_WIP_VIDEO_STREAMING; + +/** + * Initialize the video processing subsystem if the user has enabled HLS + * generation in settings. + */ +export const initVideoProcessing = async () => { + let enabled = false; + if (await savedGenerateHLS()) enabled = true; + + _state.isHLSGenerationEnabled = enabled; + + // Update snapshot to reflect the enabled setting. The estimated count will + // get filled in when we tick. + setHLSGenerationStatusSnapshot({ enabled }); +}; + +/** + * Return the persisted user preference for HLS generation. + */ +const savedGenerateHLS = () => getKVB("generateHLS"); + +/** + * Update the persisted user preference for HLS generation. + * + * Use {@link savedGenerateHLS} to get the persisted value back. + */ +const saveGenerateHLS = (enabled: boolean) => setKV("generateHLS", enabled); + /** * Enable or disable (toggle) the HLS generation on this client. * * When HLS generation is enabled, this client will process videos to generate a * streamable variant of them. * - * It can only be enabled when + * Precondition: {@link isHLSGenerationSupported} must be `true`. */ -export const toggleHLSGeneration = () => { +export const toggleHLSGeneration = async () => { if (!isHLSGenerationSupported()) { assertionFailed(); return; @@ -210,8 +243,13 @@ export const toggleHLSGeneration = () => { const enabled = !_state.isHLSGenerationEnabled; - // Right now we only set the enabled setting. The estimated count status - // will get filled in when we tick. + // Update disk. + await saveGenerateHLS(enabled); + // Update in memory. + _state.isHLSGenerationEnabled = enabled; + + // Update snapshot. Right now we only set the enabled setting. The estimated + // count will get filled in when we tick. setHLSGenerationStatusSnapshot({ enabled }); // Wake up the processor if needed. From 96d748dc87fc2d0bba6fda84b113056104b4b0e0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 15:03:03 +0530 Subject: [PATCH 08/12] status --- web/packages/gallery/services/video.ts | 54 ++++++++++++++----- .../new/photos/components/SearchBar.tsx | 27 +++++++--- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index 29300f79be..e08d4c8b13 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -49,9 +49,11 @@ import { type TimestampedFileSystemUploadItem, } from "./upload"; +export type HLSGenerationEnabledStatus = "processing" | "idle"; + export type HLSGenerationStatus = | { enabled: false } - | { enabled: true; estimatedPendingCount?: number }; + | { enabled: true; status?: HLSGenerationEnabledStatus }; interface VideoProcessingQueueItem { /** @@ -93,6 +95,11 @@ class VideoState { * {@link hlsGenerationStatusSnapshot} function. */ hlsGenerationStatusSnapshot: HLSGenerationStatus | undefined; + /** + * Value of the {@link status} field in the last + * {@link hlsGenerationStatusSnapshot}. + */ + lastEnabledStatus: HLSGenerationEnabledStatus | undefined; /** * Queue of recently uploaded items waiting to be processed. */ @@ -180,6 +187,21 @@ const setHLSGenerationStatusSnapshot = (snapshot: HLSGenerationStatus) => { _state.hlsGenerationStatusListeners.forEach((l) => l()); }; +/** + * A variant of {@link setHLSGenerationStatusSnapshot} that only triggers an + * update of the snapshot if the enabled state is different from the last known + * enabled state. + */ +const updateSnapshotIfNeeded = ( + status: HLSGenerationEnabledStatus | undefined, +) => { + const enabled = _state.isHLSGenerationEnabled; + if (enabled && status != _state.lastEnabledStatus) { + _state.lastEnabledStatus = status; + setHLSGenerationStatusSnapshot({ enabled, status }); + } +}; + /** * Return `true` if this client is capable of generating HLS streams for * uploaded videos. @@ -210,8 +232,8 @@ export const initVideoProcessing = async () => { _state.isHLSGenerationEnabled = enabled; - // Update snapshot to reflect the enabled setting. The estimated count will - // get filled in when we tick. + // Update snapshot to reflect the enabled setting. The status will get + // filled in when we tick. setHLSGenerationStatusSnapshot({ enabled }); }; @@ -243,13 +265,16 @@ export const toggleHLSGeneration = async () => { const enabled = !_state.isHLSGenerationEnabled; + // Clear transient fields. + _state.lastEnabledStatus = undefined; + // Update disk. await saveGenerateHLS(enabled); // Update in memory. _state.isHLSGenerationEnabled = enabled; - // Update snapshot. Right now we only set the enabled setting. The estimated - // count will get filled in when we tick. + // Update snapshot. Right now we only set the enabled setting. The status + // will get filled in when we tick. setHLSGenerationStatusSnapshot({ enabled }); // Wake up the processor if needed. @@ -672,7 +697,7 @@ export const videoPrunePermanentlyDeletedFileIDsIfNeeded = async ( */ export const videoProcessingSyncIfNeeded = async () => { if (!isHLSGenerationSupported()) return; - if (!isVideoProcessingEnabled()) return; + if (!isHLSGenerationEnabled()) return; await syncProcessedFileIDs(); _state.haveSyncedOnce = true; @@ -708,7 +733,7 @@ export const processVideoNewUpload = ( processableUploadItem: ProcessableUploadItem, ) => { if (!isHLSGenerationSupported()) return; - if (!isVideoProcessingEnabled()) return; + if (!isHLSGenerationEnabled()) return; if (file.metadata.fileType !== FileType.video) return; if (processableUploadItem instanceof File) { // While the types don't guarantee it, we really shouldn't be getting @@ -753,10 +778,7 @@ const tickNow = () => { _state.queueProcessor ??= processQueue(); }; -export const isVideoProcessingEnabled = () => - // TODO(HLS): - process.env.NEXT_PUBLIC_ENTE_WIP_VIDEO_STREAMING && - settingsSnapshot().isInternalUser; +export const isHLSGenerationEnabled = () => _state.isHLSGenerationEnabled; /** * The video processing loop goes through videos one by one, preferring items in @@ -795,7 +817,7 @@ export const isVideoProcessingEnabled = () => * batches, and the externally triggered processing of live uploads. */ const processQueue = async () => { - if (!isHLSGenerationSupported() || !isVideoProcessingEnabled()) { + if (!isHLSGenerationSupported() || !isHLSGenerationEnabled()) { assertionFailed(); /* we shouldn't have come here */ return; } @@ -803,7 +825,7 @@ const processQueue = async () => { const userID = ensureLocalUser().id; let bq: typeof _state.liveQueue | undefined; - while (isVideoProcessingEnabled()) { + while (isHLSGenerationEnabled()) { let item = _state.liveQueue.shift(); if (!item) { // Initialize or refill queue. @@ -818,6 +840,8 @@ const processQueue = async () => { if (bq?.length) item = bq.pop(); } if (item) { + updateSnapshotIfNeeded("processing"); + try { await processQueueItem(item); await markProcessedVideoFileID(item.file.id); @@ -831,6 +855,8 @@ const processQueue = async () => { // There are no more items in either the live queue or backlog. // Go to sleep (for increasingly longer durations, capped at a // maximum). + updateSnapshotIfNeeded("idle"); + const idleWait = _state.idleWait; _state.idleWait = Math.min(idleWait * 2, idleWaitMax); @@ -844,6 +870,8 @@ const processQueue = async () => { } } + updateSnapshotIfNeeded(undefined); + _state.queueProcessor = undefined; }; diff --git a/web/packages/new/photos/components/SearchBar.tsx b/web/packages/new/photos/components/SearchBar.tsx index db1297e3f2..8c9dd6acd6 100644 --- a/web/packages/new/photos/components/SearchBar.tsx +++ b/web/packages/new/photos/components/SearchBar.tsx @@ -13,10 +13,10 @@ import { useTheme, type Theme, } from "@mui/material"; -import { assertionFailed } from "ente-base/assert"; import { EnteLogo, EnteLogoBox } from "ente-base/components/EnteLogo"; import type { ButtonishProps } from "ente-base/components/mui"; import { useIsSmallWidth } from "ente-base/components/utils/hooks"; +import { pt } from "ente-base/i18n"; import { ItemCard, PreviewItemTile } from "ente-new/photos/components/Tiles"; import { isMLSupported, mlStatusSnapshot } from "ente-new/photos/services/ml"; import { searchOptionsForString } from "ente-new/photos/services/search"; @@ -38,6 +38,7 @@ import AsyncSelect from "react-select/async"; import { SearchPeopleList } from "./PeopleList"; import { UnstyledButton } from "./UnstyledButton"; import { + useHLSGenerationStatusSnapshot, useMLStatusSnapshot, usePeopleStateSnapshot, } from "./utils/use-snapshot"; @@ -401,15 +402,19 @@ const EmptyState: React.FC< > = ({ onSelectPeople, onSelectPerson }) => { const mlStatus = useMLStatusSnapshot(); const people = usePeopleStateSnapshot()?.visiblePeople; - - if (!mlStatus || mlStatus.phase == "disabled") { - // The preflight check should've prevented us from coming here. - assertionFailed(); - return <>; - } + const vpStatus = useHLSGenerationStatusSnapshot(); let label: string | undefined; - switch (mlStatus.phase) { + switch (mlStatus?.phase) { + case undefined: + case "disabled": + case "done": + // If ML is not running, see if video processing is. + if (vpStatus?.enabled && vpStatus.status == "processing") { + // TODO(HLS): + label = pt("Processing videos..."); + } + break; case "scheduled": label = t("indexing_scheduled"); break; @@ -424,6 +429,12 @@ const EmptyState: React.FC< break; } + // If we're neither video processing, nor ML is enabled, then don't show the + // empty state content. + if (!label && (!mlStatus || mlStatus.phase == "disabled")) { + return <>; + } + return ( {people && people.length > 0 && ( From 93413687c9ac9fc4d423541c1adaaf1c88b61f58 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 16:10:12 +0530 Subject: [PATCH 09/12] hs --- web/packages/gallery/services/video.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index e08d4c8b13..b0c2b75f80 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -697,10 +697,17 @@ export const videoPrunePermanentlyDeletedFileIDsIfNeeded = async ( */ export const videoProcessingSyncIfNeeded = async () => { if (!isHLSGenerationSupported()) return; + + // The `haveSyncedOnce` flag tracks whether or not a sync has happened for + // the app, and is not specific to video processing. We always set it even + // if HLS generation is currently disabled so that we can immediately start + // processing the backfill if it gets video processing gets enabled during + // the app's session, without waiting for the next sync to happen. + _state.haveSyncedOnce = true; + if (!isHLSGenerationEnabled()) return; await syncProcessedFileIDs(); - _state.haveSyncedOnce = true; tickNow(); /* if not already ticking */ }; From e541f0522d93ce25e4836a3d4a4ab0cf15dfdb56 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 16:40:58 +0530 Subject: [PATCH 10/12] skip case --- .../gallery/components/viewer/data-source.ts | 32 ++++++------- web/packages/gallery/services/video.ts | 46 +++++++++++-------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/web/packages/gallery/components/viewer/data-source.ts b/web/packages/gallery/components/viewer/data-source.ts index 6c5d4a3727..8eccce3179 100644 --- a/web/packages/gallery/components/viewer/data-source.ts +++ b/web/packages/gallery/components/viewer/data-source.ts @@ -4,7 +4,7 @@ import { downloadManager } from "ente-gallery/services/download"; import { extractRawExif, parseExif } from "ente-gallery/services/exif"; import { hlsPlaylistDataForFile, - type HLSPlaylistData, + type HLSPlaylistDataForFile, } from "ente-gallery/services/video"; import type { EnteFile } from "ente-media/file"; import { fileCaption, filePublicMagicMetadata } from "ente-media/file-metadata"; @@ -452,10 +452,14 @@ const enqueueUpdates = async ( const updateVideo = ( videoURL: string | undefined, - hlsPlaylistData: HLSPlaylistData | undefined, + hlsPlaylistData: HLSPlaylistDataForFile, ) => { const videoURLD = videoURL ? { videoURL } : {}; - if (hlsPlaylistData) { + // See: [Note: Caching HLS playlist data] + // + // In brief, there are three cases: + if (typeof hlsPlaylistData == "object") { + // 1. If we have a playlist, we can cache it const { playlistURL: videoPlaylistURL, width, @@ -466,18 +470,9 @@ const enqueueUpdates = async ( createHLSPlaylistItemDataValidity(), ); } else { - // See: [Note: Caching HLS playlist data] - // - // TODO(HLS): As an optimization, we can handle the logged in vs - // public albums case separately once we have the status-diff state, - // we don't need to mark status-diff case as transient. - // - // Note that setting the transient flag is not too expensive, since - // the underlying videoURL is still cached by the download manager. - // So effectively, under normal circumstance, it just adds one API - // call (to recheck if an HLS playlist now exists for the given - // file). - update({ ...videoURLD, isTransient: true }); + // 2. if the file is not eligible ("skip"), we can cache it. + // 3. Otherwise we shouldn't cache it indefinitely. + update({ ...videoURLD, isTransient: hlsPlaylistData != "skip" }); } }; @@ -526,7 +521,7 @@ const enqueueUpdates = async ( } try { - let hlsPlaylistData: HLSPlaylistData | undefined; + let hlsPlaylistData: HLSPlaylistDataForFile; if (file.metadata.fileType == FileType.video) { hlsPlaylistData = await hlsPlaylistDataForFile( file, @@ -534,7 +529,10 @@ const enqueueUpdates = async ( ); // We have a HLS playlist, and the user didn't request the original. // Early return so that we don't initiate a fetch for the original. - if (hlsPlaylistData && opts?.videoQuality != "original") { + if ( + typeof hlsPlaylistData == "object" && + opts?.videoQuality != "original" + ) { updateVideo(undefined, hlsPlaylistData); return; } diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index b0c2b75f80..06364d42fa 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -290,6 +290,11 @@ export interface HLSPlaylistData { height: number; } +/** + * See: [Note: Caching HLS playlist data] for the semantics of "skip". + */ +export type HLSPlaylistDataForFile = HLSPlaylistData | "skip" | undefined; + /** * Return a HLS playlist that can be used to stream playback of then given video * {@link file}. @@ -323,34 +328,35 @@ export interface HLSPlaylistData { * regenerated again). All in all, this means that a positive result ("this * file has a playlist") can be cached indefinitely. * - * - If a file does not have a HLS playlist, and it is eligible for being - * streamed (e.g. it is not too small where the streaming overhead is not - * required), then a client (this one, or a different one) can process it at - * any arbitrary time. So the negative result ("this file does not have a + * - If a file does not have a HLS playlist, it might be because it is not + * eligible for being streamed (e.g. it is already small and in a compatible + * codec). See [Note: Marking files which do not need video processing] for + * more details of this case. In particular, we can cache this state in memory + * indefinitely too, since there isn't a current case where either this + * eligibility can change, or the client gain the ability to handle them + * without restarting. + * + * - Finally, if a file does not have an HLS playlist but and it is eligible for + * being streamed, then a client (this one, or a different one) can process it + * at any arbitrary time. So the negative result ("this file does not have a * playlist") cannot be cached. * - * So while we can easily cache the first case ("this file has a playlist"), we - * need to deal with the second case ("this file does not have a playlist") a - * bit more intricately: - * - * - If running in the context of a logged in user (e.g. photos app), we can use - * the "/files/data/status-diff" API to be notified of any modifications to - * the second case for the user's own files. This status-diff happens during - * the regular "sync", and we can use that as a cue to selectively prune cache - * entries for the second case (but can otherwise indefinitely cache it). - * - * - If the file is a shared file, the status-diff will not return it. And if - * we're not running in the context of a logged in user (e.g. the public - * albums app), then there is no status-diff to do. For these two scenarios, - * we thus mark the cached values as "transient" and always recheck for a - * playlist when opening the slide. + * So while we can easily cache the first case ("this file has a playlist") and + * second case ("this file doesn't need a streaming variant"), we need to deal + * with the third case ("this file does not have a playlist") by marking the + * cached values as "transient" and always recheck for a playlist when opening + * the slide. */ export const hlsPlaylistDataForFile = async ( file: EnteFile, publicAlbumsCredentials?: PublicAlbumsCredentials, -): Promise => { +): Promise => { ensurePrecondition(file.metadata.fileType == FileType.video); + if (filePublicMagicMetadata(file)?.sv == 1) { + return "skip"; + } + const playlistFileData = await fetchFileData( "vid_preview", file.id, From 5b9b328c997c1c38f9ba71f1d5ba34686793e035 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 16:49:29 +0530 Subject: [PATCH 11/12] Tweak --- web/apps/photos/src/components/Sidebar.tsx | 1 - web/packages/gallery/components/viewer/data-source.ts | 2 +- web/packages/gallery/services/video.ts | 5 ++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar.tsx b/web/apps/photos/src/components/Sidebar.tsx index dd51591057..4690889d44 100644 --- a/web/apps/photos/src/components/Sidebar.tsx +++ b/web/apps/photos/src/components/Sidebar.tsx @@ -821,7 +821,6 @@ const Preferences: React.FC = ({ onClick={showAdvancedSettings} /> {isHLSGenerationSupported() && ( - // TODO(HLS): Visual look }> {t("labs")} diff --git a/web/packages/gallery/components/viewer/data-source.ts b/web/packages/gallery/components/viewer/data-source.ts index 8eccce3179..1cf614d4c4 100644 --- a/web/packages/gallery/components/viewer/data-source.ts +++ b/web/packages/gallery/components/viewer/data-source.ts @@ -471,7 +471,7 @@ const enqueueUpdates = async ( ); } else { // 2. if the file is not eligible ("skip"), we can cache it. - // 3. Otherwise we shouldn't cache it indefinitely. + // 3. Otherwise it's transient and shouldn't be cached indefinitely. update({ ...videoURLD, isTransient: hlsPlaylistData != "skip" }); } }; diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index 06364d42fa..a17489328f 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -178,9 +178,8 @@ export const hlsGenerationStatusSubscribe = ( * A return value of `undefined` indicates that the HLS generation subsystem has * not been initialized yet. */ -export const hlsGenerationStatusSnapshot = (): - | HLSGenerationStatus - | undefined => _state.hlsGenerationStatusSnapshot; +export const hlsGenerationStatusSnapshot = () => + _state.hlsGenerationStatusSnapshot; const setHLSGenerationStatusSnapshot = (snapshot: HLSGenerationStatus) => { _state.hlsGenerationStatusSnapshot = snapshot; From b4a60fd2f4dacd2ee107a54ba80b2058621c586c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 13 May 2025 17:17:38 +0530 Subject: [PATCH 12/12] empty state --- web/packages/gallery/services/video.ts | 4 +++ .../new/photos/components/SearchBar.tsx | 31 +++++++++++++++---- web/packages/new/photos/services/ml/index.ts | 6 ++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.ts index a17489328f..75021e7248 100644 --- a/web/packages/gallery/services/video.ts +++ b/web/packages/gallery/services/video.ts @@ -175,6 +175,10 @@ export const hlsGenerationStatusSubscribe = ( * * See also {@link hlsGenerationStatusSubscribe}. * + * This function can be safely called even if {@link isHLSGenerationSupported} + * is `false` (in such cases, it will always return `undefined`). This is so + * that it can be unconditionally called as part of a React hook. + * * A return value of `undefined` indicates that the HLS generation subsystem has * not been initialized yet. */ diff --git a/web/packages/new/photos/components/SearchBar.tsx b/web/packages/new/photos/components/SearchBar.tsx index 8c9dd6acd6..0715727789 100644 --- a/web/packages/new/photos/components/SearchBar.tsx +++ b/web/packages/new/photos/components/SearchBar.tsx @@ -17,6 +17,10 @@ import { EnteLogo, EnteLogoBox } from "ente-base/components/EnteLogo"; import type { ButtonishProps } from "ente-base/components/mui"; import { useIsSmallWidth } from "ente-base/components/utils/hooks"; import { pt } from "ente-base/i18n"; +import { + hlsGenerationStatusSnapshot, + isHLSGenerationSupported, +} from "ente-gallery/services/video"; import { ItemCard, PreviewItemTile } from "ente-new/photos/components/Tiles"; import { isMLSupported, mlStatusSnapshot } from "ente-new/photos/services/ml"; import { searchOptionsForString } from "ente-new/photos/services/search"; @@ -383,11 +387,26 @@ const shouldShowEmptyState = (inputValue: string) => { // Don't show empty state if the user has entered search input. if (inputValue) return false; - // Don't show empty state if there is no ML related information. - if (!isMLSupported) return false; + // Don't show empty state if there is no ML related information AND we're + // not processing videos. - const status = mlStatusSnapshot(); - if (!status || status.phase == "disabled") return false; + if (!isMLSupported && !isHLSGenerationSupported()) { + // Neither of ML or HLS generation is supported on current client. This + // is the code path for web. + return false; + } + + const mlStatus = mlStatusSnapshot(); + const vpStatus = hlsGenerationStatusSnapshot(); + if ( + (!mlStatus || mlStatus.phase == "disabled") && + (!vpStatus?.enabled || vpStatus.status != "processing") + ) { + // ML is either not supported or currently disabled AND video processing + // is either not supported or currently not happening. Don't show the + // empty state. + return false; + } // Show it otherwise. return true; @@ -429,9 +448,9 @@ const EmptyState: React.FC< break; } - // If we're neither video processing, nor ML is enabled, then don't show the + // If ML is disabled and we're not video processing, then don't show the // empty state content. - if (!label && (!mlStatus || mlStatus.phase == "disabled")) { + if ((!mlStatus || mlStatus.phase == "disabled") && !label) { return <>; } diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index f97d9d91bd..f02078a99b 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -471,10 +471,16 @@ export const mlStatusSubscribe = (onChange: () => void): (() => void) => { * * See also {@link mlStatusSubscribe}. * + * This function can be safely called even if {@link isMLSupported} is `false` + * (in such cases, it will always return `undefined`). This is so that it can be + * unconditionally called as part of a React hook. + * * A return value of `undefined` indicates that we're still performing the * asynchronous tasks that are needed to get the status. */ export const mlStatusSnapshot = (): MLStatus | undefined => { + if (!isMLSupported) return undefined; + const result = _state.mlStatusSnapshot; // We don't have it yet, trigger an update. if (!result) triggerStatusUpdate();