[web] Video streaming WIP - Part 1/x (#5359)

web/docs/dependencies.md

@@ -133,8 +133,10 @@ For showing the app's UI in multiple languages, we use the
 [i18next](https://www.i18next.com), specifically its three components
 
 - [i18next](https://github.com/i18next/i18next): The core `i18next` library.
+
 - [react-i18next](https://github.com/i18next/react-i18next): React specific
   support in `i18next`.
+
 - [i18next-http-backend](https://github.com/i18next/i18next-http-backend): Adds
   support for initializing `i18next` with JSON file containing the translation
   in a particular language, fetched at runtime.
@@ -173,7 +175,9 @@ via [@fontsource-variable/inter](https://fontsource.org/fonts/inter/install).
   layer on top of web workers to make them more easier to use.
 
 - [idb](https://github.com/jakearchibald/idb) provides a promise API over the
-  browser-native IndexedDB APIs.
+  browser-native IndexedDB APIs. Older code (the file and collection store),
+  uses [localForage](https://github.com/localForage/localForage) for IndexedDB
+  access.
 
     > For more details about IDB and its role, see [storage.md](storage.md).
 

web/docs/storage.md

@@ -41,6 +41,7 @@ For more details, see:
 
 - https://web.dev/articles/indexeddb
 - https://github.com/jakearchibald/idb
+- https://github.com/localForage/localForage
 
 ## OPFS
 

web/packages/gallery/components/viewer/FileViewer.tsx

@@ -8,6 +8,7 @@ import { LoadingButton } from "@/base/components/mui/LoadingButton";
 import { useIsSmallWidth } from "@/base/components/utils/hooks";
 import { type ModalVisibilityProps } from "@/base/components/utils/modal";
 import { useBaseContext } from "@/base/context";
+import { isDevBuild } from "@/base/env";
 import { lowercaseExtension } from "@/base/file-name";
 import { formattedListJoin, ut } from "@/base/i18n";
 import type { LocalUser } from "@/base/local-user";
@@ -53,6 +54,7 @@ import React, {
     useRef,
     useState,
 } from "react";
+import { hlsPlaylistForFile } from "../../services/video";
 import {
     fileInfoExifForFile,
     updateItemDataAlt,
@@ -518,6 +520,15 @@ export const FileViewer: React.FC<FileViewerProps> = ({
                 }
             })();
 
+            if (
+                isDevBuild &&
+                process.env.NEXT_PUBLIC_ENTE_WIP_VIDEO_STREAMING
+            ) {
+                if (file.metadata.fileType == FileType.video) {
+                    void hlsPlaylistForFile(file);
+                }
+            }
+
             const annotation: FileViewerFileAnnotation = {
                 fileID,
                 isOwnFile,

web/packages/gallery/services/file-data.ts

@@ -18,7 +18,9 @@ import { z } from "zod";
  * There are specialized APIs for fetching and uploading the originals and the
  * thumbnails. But for the other associated data, we can use the file data APIs.
  */
-type FileDataType = "mldata" /* See: [Note: "mldata" format] */;
+type FileDataType =
+    | "mldata" /* See: [Note: "mldata" format] */
+    | "vid_preview" /* See: [Note: Video playlist and preview] */;
 
 const RemoteFileData = z.object({
     /**
@@ -61,7 +63,7 @@ type RemoteFileData = z.infer<typeof RemoteFileData>;
  * payload, but we don't parse that information currently since the higher
  * levels of our code that use this function handle such rare skips gracefully.
  */
-export const fetchFileData = async (
+export const fetchFilesData = async (
     type: FileDataType,
     fileIDs: number[],
 ): Promise<RemoteFileData[]> => {
@@ -75,6 +77,27 @@ export const fetchFileData = async (
         .data;
 };
 
+/**
+ * A variant of {@link fetchFilesData} that fetches data for a single file.
+ *
+ * Unlike {@link fetchFilesData}, this uses a HTTP GET request.
+ *
+ * Returns `undefined` if no video preview has been generated for this file yet.
+ */
+export const fetchFileData = async (
+    type: FileDataType,
+    fileID: number,
+): Promise<RemoteFileData | undefined> => {
+    const params = new URLSearchParams({ type, fileID: fileID.toString() });
+    const url = await apiURL("/files/data/fetch");
+    const res = await fetch(`${url}?${params.toString()}`, {
+        headers: await authenticatedRequestHeaders(),
+    });
+    if (res.status == 404) return undefined;
+    ensureOk(res);
+    return z.object({ data: RemoteFileData }).parse(await res.json()).data;
+};
+
 /**
  * Upload file data associated with the given file to remote.
  *

web/packages/gallery/services/video.ts

@@ -0,0 +1,96 @@
+import { decryptBlob } from "@/base/crypto";
+import type { EncryptedBlob } from "@/base/crypto/types";
+import log from "@/base/log";
+import type { EnteFile } from "@/media/file";
+import { FileType } from "@/media/file-type";
+import { gunzip } from "@/new/photos/utils/gzip";
+import { ensurePrecondition } from "@/utils/ensure";
+import { z } from "zod";
+import { fetchFileData } from "./file-data";
+
+/**
+ * Return a HLS playlist that can be used to stream playback of thne given video
+ * {@link file}.
+ *
+ * @param file An {@link EnteFile} of type video.
+ *
+ * @returns The HLS playlist as a string, or `undefined` if there is no video
+ * preview associated with the given file.
+ *
+ * [Note: Video playlist and preview]
+ *
+ * In museum's ontology, there is a distinction between two concepts:
+ *
+ * S3 metadata is the data that museum uploads (on behalf of the client):
+ * - ML data.
+ * - Preview video playlist.
+ *
+ * S3 file data is the data that client uploads:
+ * - Preview video itself.
+ * - Additional preview images.
+ *
+ * Because of this separation, there are separate code paths dealing with the
+ * two parts we need to play streaming video:
+ *
+ * - The encrypted HLS playlist (which is stored as file data of type
+ *   "vid_preview"),
+ *
+ * - And the encrypted video chunks that it (the playlist) refers to (which are
+ *   stored as file preview data of type "vid_preview").
+ */
+export const hlsPlaylistForFile = async (file: EnteFile) => {
+    ensurePrecondition(file.metadata.fileType == FileType.video);
+
+    const playlistFileData = await fetchFileData("vid_preview", file.id);
+    if (!playlistFileData) return undefined;
+
+    // See: [Note: strict mode migration]
+    //
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    const { playlist } = await decryptPlaylistJSON(playlistFileData, file);
+
+    // [Note: HLS playlist format]
+    //
+    // The decrypted playlist is a regular HLS playlist for an encrypted media
+    // stream, except that it uses a placeholder "output.ts" which needs to be
+    // replaced with the URL of the actual encrypted video data. A single URL
+    // pointing to the entire encrypted video data suffices; the individual
+    // chunks are fetched by HTTP range requests.
+    //
+    // Here is an example of what the contents of the `playlist` variable might
+    // look like at this point:
+    //
+    //     #EXTM3U
+    //     #EXT-X-VERSION:4
+    //     #EXT-X-TARGETDURATION:8
+    //     #EXT-X-MEDIA-SEQUENCE:0
+    //     #EXT-X-KEY:METHOD=AES-128,URI="data:text/plain;base64,XjvG7qeRrsOpPUbJPh2Ikg==",IV=0x00000000000000000000000000000000
+    //     #EXTINF:8.333333,
+    //     #EXT-X-BYTERANGE:3046928@0
+    //     output.ts
+    //     #EXTINF:8.333333,
+    //     #EXT-X-BYTERANGE:3012704@3046928
+    //     output.ts
+    //     #EXTINF:2.200000,
+    //     #EXT-X-BYTERANGE:834736@6059632
+    //     output.ts
+    //     #EXT-X-ENDLIST
+    //
+    log.debug(() => ["hlsPlaylistForFile", playlist]);
+    return file.id;
+};
+
+const PlaylistJSON = z.object({
+    /** The HLS playlist, as a string. */
+    playlist: z.string(),
+});
+
+const decryptPlaylistJSON = async (
+    encryptedPlaylist: EncryptedBlob,
+    file: EnteFile,
+) => {
+    const decryptedBytes = await decryptBlob(encryptedPlaylist, file.key);
+    const jsonString = await gunzip(decryptedBytes);
+    return PlaylistJSON.parse(JSON.parse(jsonString));
+};

web/packages/new/photos/services/ml/ml-data.ts

@@ -1,10 +1,10 @@
 import { decryptBlob } from "@/base/crypto";
 import log from "@/base/log";
+import { fetchFilesData, putFileData } from "@/gallery/services/file-data";
 import type { EnteFile } from "@/media/file";
 import { nullToUndefined } from "@/utils/transform";
 import { z } from "zod";
 import { gunzip, gzip } from "../../utils/gzip";
-import { fetchFileData, putFileData } from "../file-data";
 import { type RemoteCLIPIndex } from "./clip";
 import { type RemoteFaceIndex } from "./face";
 
@@ -153,7 +153,7 @@ const ParsedRemoteMLData = z.object({
 export const fetchMLData = async (
     filesByID: Map<number, EnteFile>,
 ): Promise<Map<number, RemoteMLData>> => {
-    const remoteFileDatas = await fetchFileData("mldata", [
+    const remoteFileDatas = await fetchFilesData("mldata", [
         ...filesByID.keys(),
     ]);
 

web/packages/utils/ensure.ts

@@ -1,3 +1,13 @@
+/**
+ * Throw an exception if the given value {@link v} is false-y.
+ *
+ * This is a variant of {@link assertionFailed}, except it always throws, not
+ * just in dev builds, if the given value is falsey.
+ */
+export const ensurePrecondition = (v: unknown): void => {
+    if (!v) throw new Error("Precondition failed");
+};
+
 /**
  * Throw an exception if the given value is not a string.
  */

web/packages/utils/tsconfig.json

@@ -1,16 +1 @@
-{
-    "extends": "@/build-config/tsconfig-typecheck.json",
-    "compilerOptions": {
-        /* Require sufficient type annotations on exports so that non-tsc tools
-           can trivially (and quickly) generate declaration files
-
-           Eventually, we'd want to enable this for all our code, but currently
-           that cannot be done because it clashes with the allowJs flag that
-           Next.js insists on adding.
-         */
-        "isolatedDeclarations": true,
-        /* Required for isolatedDeclarations */
-        "composite": true
-    },
-    "include": ["."]
-}
+{ "extends": "@/build-config/tsconfig-typecheck.json", "include": ["."] }