sketch 2

desktop/src/main/services/ffmpeg-worker.ts

@@ -16,7 +16,7 @@ import { z } from "zod";
 import type { FFmpegCommand } from "../../types/ipc";
 import log from "../log-worker";
 import { messagePortMainEndpoint } from "../utils/comlink";
-import { wait } from "../utils/common";
+import { nullToUndefined, wait } from "../utils/common";
 import { execAsyncWorker } from "../utils/exec-worker";
 import { publicRequestHeaders } from "../utils/http";
 
@@ -46,6 +46,7 @@ export interface FFmpegUtilityProcess {
     ffmpegGenerateHLSPlaylistAndSegments: (
         inputFilePath: string,
         outputPathPrefix: string,
+        fetchURL: string,
         authToken: string,
     ) => Promise<FFmpegGenerateHLSPlaylistAndSegmentsResult | undefined>;
 
@@ -207,6 +208,7 @@ export interface FFmpegGenerateHLSPlaylistAndSegmentsResult {
     playlistPath: string;
     dimensions: { width: number; height: number };
     videoSize: number;
+    videoObjectID: string;
 }
 
 /**
@@ -233,8 +235,11 @@ export interface FFmpegGenerateHLSPlaylistAndSegmentsResult {
  * the user's local file system. This function will write the generated HLS
  * playlist and video segments under this prefix.
  *
- * @param authToken A token that can be used to make API request to obtain
- * pre-signed S3 URLs for uploading the generated video segment file.
+ * @param fetchURL The fully resolved API URL for obtaining pre-signed S3 URLs
+ * for uploading the generated video segment file.
+ *
+ * @param authToken A token that can be used to make API request to
+ * {@link fetchURL}.
  *
  * @returns The path to the file on the user's file system containing the
  * generated HLS playlist, and other metadata about the generated video stream.
@@ -245,6 +250,7 @@ export interface FFmpegGenerateHLSPlaylistAndSegmentsResult {
 const ffmpegGenerateHLSPlaylistAndSegments = async (
     inputFilePath: string,
     outputPathPrefix: string,
+    fetchURL: string,
     authToken: string,
 ): Promise<FFmpegGenerateHLSPlaylistAndSegmentsResult | undefined> => {
     const { isH264, isHDR, bitrate } =
@@ -512,6 +518,7 @@ const ffmpegGenerateHLSPlaylistAndSegments = async (
 
     let dimensions: { width: number; height: number };
     let videoSize: number;
+    let videoObjectID: string;
 
     try {
         // Write the key and the keyInfo to their desired paths.
@@ -545,7 +552,12 @@ const ffmpegGenerateHLSPlaylistAndSegments = async (
         // the generated .ts file.
         videoSize = await fs.stat(videoPath).then((st) => st.size);
 
-        await uploadVideoSegments(videoPath, videoSize, authToken);
+        videoObjectID = await uploadVideoSegments(
+            videoPath,
+            videoSize,
+            fetchURL,
+            authToken,
+        );
     } catch (e) {
         log.error("HLS generation failed", e);
         await Promise.all([deletePathIgnoringErrors(playlistPath)]);
@@ -561,7 +573,7 @@ const ffmpegGenerateHLSPlaylistAndSegments = async (
         ]);
     }
 
-    return { playlistPath, dimensions, videoSize };
+    return { playlistPath, dimensions, videoSize, videoObjectID };
 };
 
 /**
@@ -808,7 +820,7 @@ const pseudoFFProbeVideo = async (inputFilePath: string) => {
 };
 
 /**
- * Upload the file at the given {@link videoFilePath} to the provided presigned
+ * Upload the file at the given {@link videoFilePath} to the provided pre-signed
  * URL(s) using a HTTP PUT request.
  *
  * All HTTP requests are retried up to 3 times with exponential backoff.
@@ -820,14 +832,18 @@ const pseudoFFProbeVideo = async (inputFilePath: string) => {
  *
  * @param videoSize The size in bytes of the file at {@link videoFilePath}.
  *
- * @param authToken The user's auth token (for fetch pre-signed upload URLs).
+ * @param fetchURL The API URL for fetching pre-signed upload URLs.
+ *
+ * @param authToken The user's auth token for use with {@link fetchURL}.
+ *
+ * @return The object ID of the uploaded file on remote storage.
  */
 const uploadVideoSegments = async (
     videoFilePath: string,
     videoSize: number,
+    fetchURL: string,
     authToken: string,
 ) => {
-
     // [Note: Passing HLS multipart upload URLs over IPC]
     //
     // For IPC convenience, we convert both normal upload URLs (where we have
@@ -877,7 +893,7 @@ const FilePreviewDataUploadURLResponse = z.object({
      */
     objectID: z.string(),
     /**
-     * A presigned URL that can be used to upload the file.
+     * A pre-signed URL that can be used to upload the file.
      *
      * This will be present only if we requested a singular object upload URL.
      */
@@ -900,7 +916,7 @@ const FilePreviewDataUploadURLResponse = z.object({
 });
 
 /**
- * Obtain a presigned URL(s) that can be used to upload the "file preview data"
+ * Obtain a pre-signed URL(s) that can be used to upload the "file preview data"
  * of type "vid_preview" (the file containing the encrypted video segments which
  * the "vid_preview" HLS playlist for the file would refer to).
  */

desktop/src/main/stream.ts

@@ -289,8 +289,9 @@ const handleGenerateHLSWrite = async (
     request: Request,
     params: URLSearchParams,
 ) => {
+    const fetchURL = params.get("fetchURL");
     const authToken = params.get("authToken");
-    if (!authToken) throw new Error("Missing auth token");
+    if (!fetchURL || !authToken) throw new Error("Missing params");
 
     let inputItem: Parameters<typeof makeFileForStreamOrPathOrZipItem>[0];
     const path = params.get("path");
@@ -324,6 +325,7 @@ const handleGenerateHLSWrite = async (
         result = await worker.ffmpegGenerateHLSPlaylistAndSegments(
             inputFilePath,
             outputFilePathPrefix,
+            fetchURL,
             authToken,
         );
 

desktop/src/main/utils/common.ts

@@ -15,3 +15,11 @@
  */
 export const wait = (ms: number) =>
     new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Convert `null` to `undefined`, passthrough everything else unchanged.
+ *
+ * Duplicated from `web/packages/utils/transform.ts`.
+ */
+export const nullToUndefined = <T>(v: T | null | undefined): T | undefined =>
+    v === null ? undefined : v;

web/packages/base/http.ts

@@ -21,7 +21,7 @@ export const authenticatedRequestHeaders = async () => ({
 /**
  * Return headers that should be passed alongwith (almost) all unauthenticated
  * `fetch` calls that we make to our remotes like our API servers (museum), or
- * to presigned URLs that are handled by the S3 storage buckets themselves.
+ * to pre-signed URLs that are handled by the S3 storage buckets themselves.
  *
  * - The client package name.
  */

web/packages/gallery/components/viewer/data-source.ts

@@ -654,10 +654,10 @@ const thumbnailDimensions = (
     return { width: thumbnailWidth, height: thumbnailHeight };
 };
 /**
- * Return a new validity for a HLS playlist containing presigned URLs.
+ * Return a new validity for a HLS playlist containing pre-signed URLs.
  *
  * The content chunks in HLS playlist generated by
- * {@link hlsPlaylistDataForFile} use presigned URLs generated by remote (see
+ * {@link hlsPlaylistDataForFile} use pre-signed URLs generated by remote (see
  * `PreSignedRequestValidityDuration` in the museum source). These have a
  * validity of 7 days. We keep a 2 day buffer, and consider any item data that
  * uses such playlist as stale after 5 days.

web/packages/gallery/services/download.ts

@@ -711,7 +711,7 @@ const photos_downloadFile = async (file: EnteFile): Promise<Response> => {
     //    credentials in the "X-Auth-Token".
     //
     // 2. The proxy then does both the original steps: (a). Use the credentials
-    //    to get the pre signed URL, and (b) fetch that pre signed URL and
+    //    to get the pre-signed URL, and (b) fetch that pre-signed URL and
     //    stream back the response.
 
     const getFile = async () => {

web/packages/gallery/services/file-data.ts

@@ -321,7 +321,7 @@ export const putFileData = async (
  * context of the public albums app. If these are not specified, then the
  * credentials of the logged in user are used.
  *
- * @returns the (presigned) URL to the preview data, or undefined if there is
+ * @returns the (pre-signed) URL to the preview data, or undefined if there is
  * not preview data of the given type for the given file yet.
  *
  * [Note: File data vs file preview data]

web/packages/gallery/services/upload/remote.ts

@@ -326,7 +326,7 @@ const createMultipartUploadRequestBody = (
  * Complete a multipart upload by reporting information about all the uploaded
  * parts to the provided {@link completionURL}.
  *
- * @param completionURL A presigned URL to which the final status of the
+ * @param completionURL A pre-signed URL to which the final status of the
  * uploaded parts should be reported to.
  *
  * @param completedParts Information about all the parts of the file that have
@@ -350,7 +350,7 @@ const createMultipartUploadRequestBody = (
  * The flow is implemented in two ways:
  *
  * a. The normal way, where each requests is made to a remote S3 bucket directly
- *    using the presigned URL.
+ *    using the pre-signed URL.
  *
  * b. Using workers, where the requests are proxied via a worker near to the
  *    user's network to speed the requests up.
@@ -360,20 +360,20 @@ const createMultipartUploadRequestBody = (
  *
  * In both cases, the overall flow is roughly like the following:
  *
- * 1. Obtain multiple presigned URLs from remote (museum). The specific API call
- *    will be different (because of the different authentication mechanisms)
- *    when we're running in the context of the photos app
+ * 1. Obtain multiple pre-signed URLs from remote (museum). The specific API
+ *    call will be different (because of the different authentication
+ *    mechanisms) when we're running in the context of the photos app
  *    ({@link fetchMultipartUploadURLs}) and when we're running in the context
  *    of the public albums app ({@link fetchPublicAlbumsMultipartUploadURLs}).
  *
  * 2. Break the file to be uploaded into parts, and upload each part using a PUT
- *    request to one of the presigned URLs we got in step 1. There are two
+ *    request to one of the pre-signed URLs we got in step 1. There are two
  *    variants of this - one where we directly upload to the remote (S3)
  *    ({@link putFilePart}), and one where we go via a worker
  *    ({@link putFilePartViaWorker}).
  *
  * 3. Once all the parts have been uploaded, send a consolidated report of all
- *    the uploaded parts (the step 2's) to remote via another presigned
+ *    the uploaded parts (the step 2's) to remote via another pre-signed
  *    "completion URL" that we also got in step 1. Like step 2, there are 2
  *    variants of this - one where we directly tell the remote (S3)
  *    ({@link completeMultipartUpload}), and one where we report via a worker

web/packages/gallery/services/video.ts

@@ -11,6 +11,7 @@ import {
 import { getKV, getKVB, getKVN, setKV } from "ente-base/kv";
 import { ensureAuthToken, ensureLocalUser } from "ente-base/local-user";
 import log from "ente-base/log";
+import { apiURL } from "ente-base/origins";
 import { fileLogID, type EnteFile } from "ente-media/file";
 import {
     filePublicMagicMetadata,
@@ -318,7 +319,7 @@ export type HLSPlaylistDataForFile = HLSPlaylistData | "skip" | undefined;
  * - If a file has a corresponding HLS playlist, then currently there is no
  *   scenario (apart from file deletion, where the playlist also gets deleted)
  *   where the playlist is deleted after being created. There is a limit to the
- *   validity of the presigned chunk URLs within the playlist we create (which
+ *   validity of the pre-signed chunk URLs within the playlist we create (which
  *   we do handle, see `createHLSPlaylistItemDataValidity`), but the original
  *   playlist itself does not change. Updates are technically possible, but
  *   apart from a misbehaving client, are not expected (and should be no-ops in
@@ -998,7 +999,7 @@ const processQueueItem = async ({
     // duplicate the stream beforehand, which invalidates the point of
     // streaming.
     //
-    // Another mid-way option was to do it partially here - obtain the presigned
+    // Another mid-way option was to do it partially here - obtain the pre-signed
     // upload URLs here (since we already have the rest of the scaffolding to
     // make API requests), and then provide this pre-signed URL to the node side
     // so that it can directly upload the generated video segments.
@@ -1016,15 +1017,21 @@ const processQueueItem = async ({
     // the desktop app is more simple and straightforward (at the cost of
     // needing set up of some API request scaffolding on the desktop side).
     //
-    // We also need to pass the auth token to allow the desktop app to make the
-    // API request.
+    // Below we prepare the things that we need to pass to the desktop app to
+    // allow it to make the API request for obtaining pre-signed upload URLs.
+    const fetchURL = await apiURL("/files/data/preview-upload-url");
     const authToken = await ensureAuthToken();
 
     log.info(`Generate HLS for ${fileLogID(file)} | start`);
 
     let res: GenerateHLSResult | undefined;
     try {
-        res = await initiateGenerateHLS(electron, sourceVideo, authToken);
+        res = await initiateGenerateHLS(
+            electron,
+            sourceVideo,
+            fetchURL,
+            authToken,
+        );
     } catch (e) {
         // Failures during stream generation on the native side are expected to
         // happen in two cases:
@@ -1048,7 +1055,7 @@ const processQueueItem = async ({
         return;
     }
 
-    const { playlistToken, dimensions, videoSize } = res;
+    const { playlistToken, dimensions, videoSize, videoObjectID } = res;
     try {
         const playlist = await readVideoStream(electron, playlistToken).then(
             (res) => res.text(),
@@ -1063,7 +1070,8 @@ const processQueueItem = async ({
 
         try {
             await retryAsyncOperation(
-                () => putVideoData(file, playlistData, objectID, videoSize),
+                () =>
+                    putVideoData(file, playlistData, videoObjectID, videoSize),
                 { retryProfile: "background" },
             );
         } catch (e) {

web/packages/gallery/utils/native-stream.ts

@@ -165,6 +165,10 @@ const GenerateHLSResult = z.object({
      * The size (in bytes) of the file containing the encrypted video segments.
      */
     videoSize: z.number(),
+    /**
+     * The ID of the uploaded encrypted video segment file on the remote bucket.
+     */
+    videoObjectID: z.string(),
 });
 
 export type GenerateHLSResult = z.infer<typeof GenerateHLSResult>;
@@ -189,12 +193,14 @@ export type GenerateHLSResult = z.infer<typeof GenerateHLSResult>;
  *
  * - Otherwise it should be a {@link ReadableStream} of the video contents.
  *
- * @param authToken The user's auth token (needed to make API requests to obtain
- * the pre-signed URLs where the video segments should be uploaded to).
+ * @param fetchURL The fully resolved API URL for obtaining the pre-signed URLs
+ * to which the video segment file should be uploaded.
+ *
+ * @param authToken The user's auth token (for making the request to
+ * {@link fetchURL}).
  *
  * @returns a token that can be used to retrieve the generated HLS playlist, and
- * metadata about the generated video (its byte size and dimensions). See {@link
- * GenerateHLSResult}.
+ * metadata about the generated video (See {@link GenerateHLSResult}).
  *
  * In case the video is such that it doesn't require a separate stream to be
  * generated (e.g. it is a small video using an already compatible codec), then
@@ -205,9 +211,14 @@ export type GenerateHLSResult = z.infer<typeof GenerateHLSResult>;
 export const initiateGenerateHLS = async (
     _: Electron,
     video: FileSystemUploadItem | ReadableStream,
+    fetchURL: string,
     authToken: string,
 ): Promise<GenerateHLSResult | undefined> => {
-    const params = new URLSearchParams({ op: "generate-hls", authToken });
+    const params = new URLSearchParams({
+        op: "generate-hls",
+        fetchURL,
+        authToken,
+    });
 
     let body: ReadableStream | null;
     if (video instanceof ReadableStream) {