TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework 1

Browse Source 
temp files will need to be handled on main process
This commit is contained in:
Manav Rathi
2025-05-07 12:36:55 +05:30
parent ae925a240e
commit bd8fc08b7c
9 changed files with 151 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
desktop/src/main/services/ffmpeg-worker.ts
View File

@@ -24,17 +24,41 @@ const ffmpegPathPlaceholder = "FFMPEG";
const inputPathPlaceholder = "INPUT";
const outputPathPlaceholder = "OUTPUT";
/**
* The interface of the object exposed by `ffmpeg-worker.ts` on the message port
* pair that the main process creates to communicate with it.
*
* @see {@link ffmpegUtilityProcessPort}.
*/
export interface FFmpegUtilityProcess {
ffmpegExec: (
command: FFmpegCommand,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
) => Promise<Uint8Array>;
ffmpegConvertToMP4: (
inputFilePath: string,
outputFilePath: string,
) => Promise<void>;
ffmpegGenerateHLSPlaylistAndSegments: (
inputFilePath: string,
outputPathPrefix: string,
) => Promise<FFmpegGenerateHLSPlaylistAndSegmentsResult | undefined>;
}
log.debugString("Started ffmpeg utility process");
process.parentPort.once("message", (e) => {
// Expose an instance of `ElectronFFmpegWorker & ElectronFFmpegWorkerNode`
// on the port we got from our parent.
// Expose an instance of `FFmpegUtilityProcess` on the port we got from our
// parent.
expose(
{
ffmpegExec,
ffmpegConvertToMP4,
ffmpegGenerateHLSPlaylistAndSegments,
},
} satisfies FFmpegUtilityProcess,
messagePortMainEndpoint(e.ports[0]!),
);
});

14
desktop/src/main/services/ffmpeg.ts Normal file
View File

@@ -0,0 +1,14 @@
/**
* @file A bridge to the ffmpeg utility process. This code runs in the main
* process.
*/
import type { FFmpegUtilityProcess } from "./ffmpeg-worker";
import { ffmpegUtilityProcessPort } from "./workers";
/**
* Return a handle to the ffmpeg utility process, starting it if needed.
*/
export const ffmpegUtilityProcess = () => {
return ffmpegUtilityProcessPort() as unknown as FFmpegUtilityProcess;
};

118
desktop/src/main/services/workers.ts
View File

@@ -8,26 +8,28 @@ import {
type BrowserWindow,
type UtilityProcess,
} from "electron";
import { app, utilityProcess } from "electron/main";
import { app, utilityProcess, type MessagePortMain } from "electron/main";
import path from "node:path";
import type { UtilityProcessType } from "../../types/ipc";
import log, { processUtilityProcessLogMessage } from "../log";
import type { FFmpegGenerateHLSPlaylistAndSegmentsResult } from "./ffmpeg-worker";
/** The active ML utility process, if any. */
let _childML: UtilityProcess | undefined;
let _utilityProcessML: UtilityProcess | undefined;
/** The active ffmpeg utility process, if any. */
let _childFFmpeg: UtilityProcess | undefined;
/**
* A {@link MessagePort} that can be used to communicate with
* the active ffmpeg utility process (if any).
*/
let _utilityProcessFFmpegPort: MessagePortMain | undefined;
/**
* Create a new utility process of the given {@link type}, terminating the older
* ones (if any).
*
* The following note explains the reasoning why utility processes were used for
* the first workload (ML) that was handled this way. Similar reasoning applies
* to subsequent workloads (ffmpeg) that have been offloaded to utility
* processes to avoid stutter in the UI.
* Currently the only type is "ml". The following note explains the reasoning
* why utility processes were used for the first workload (ML) that was handled
* this way. Similar reasoning applies to subsequent workloads (ffmpeg) that
* have been offloaded to utility processes to avoid stutter in the UI.
*
* [Note: ML IPC]
*
@@ -85,22 +87,13 @@ let _childFFmpeg: UtilityProcess | undefined;
export const triggerCreateUtilityProcess = (
type: UtilityProcessType,
window: BrowserWindow,
) => {
switch (type) {
case "ml":
triggerCreateMLUtilityProcess(window);
break;
case "ffmpeg":
triggerCreateFFmpegUtilityProcess(window);
break;
}
};
) => triggerCreateMLUtilityProcess(window);
export const triggerCreateMLUtilityProcess = (window: BrowserWindow) => {
if (_childML) {
if (_utilityProcessML) {
log.debug(() => "Terminating previous ML utility process");
_childML.kill();
_childML = undefined;
_utilityProcessML.kill();
_utilityProcessML = undefined;
}
const { port1, port2 } = new MessageChannelMain();
@@ -113,7 +106,7 @@ export const triggerCreateMLUtilityProcess = (window: BrowserWindow) => {
handleMessagesFromMLUtilityProcess(child);
_childML = child;
_utilityProcessML = child;
};
/**
@@ -148,27 +141,45 @@ const handleMessagesFromMLUtilityProcess = (child: UtilityProcess) => {
});
};
export const triggerCreateFFmpegUtilityProcess = (window: BrowserWindow) => {
if (_childFFmpeg) {
log.debug(() => "Terminating previous ffmpeg utility process");
_childFFmpeg.kill();
_childFFmpeg = undefined;
}
/**
* A port that can be used to communicate with the ffmpeg utility process. If
* there is no ffmpeg utility process, a new one is created on demand.
*
* See [Note: ML IPC] for a general outline of why utility processes are needed
* (tl;dr; to avoid stutter on the UI).
*
* In the case of ffmpeg, the IPC flow is a bit different: the utility process
* is not exposed to the web layer, and is internal to the node layer. The
* reason for this difference is that we need to create temporary files etc, and
* doing it a utility process requires access to the `app` module which are not
* accessible (See: [Note: Using Electron APIs in UtilityProcess]).
*
* There could've been possible reasonable workarounds, but the architecture
* we've adopted of three layers:
*
* Renderer (web) <-> Node.js main <-> Node.js ffmpeg utility process
*
* The temporary file creation etc is handled in the Node.js main process, and
* paths to the files are forwarded to the ffmpeg utility process to act on.
*
* @returns a port that can be used to communiate with the utility process. The
* utility process is expected to expose an object that conforms to the
* {@link ElectronFFmpegWorkerNode} interface on this port.
*/
export const ffmpegUtilityProcessPort = () => {
if (_utilityProcessFFmpegPort) return _utilityProcessFFmpegPort;
const { port1, port2 } = new MessageChannelMain();
const child = utilityProcess.fork(path.join(__dirname, "ffmpeg-worker.js"));
// TODO
const userDataPath = app.getPath("userData");
child.postMessage({ userDataPath }, [port1]);
window.webContents.postMessage("utilityProcessPort/ffmpeg", undefined, [
port2,
]);
// Send a handle to the port
child.postMessage({}, [port1]);
handleMessagesFromFFmpegUtilityProcess(child);
_childFFmpeg = child;
_utilityProcessFFmpegPort = port2;
return _utilityProcessFFmpegPort;
};
const handleMessagesFromFFmpegUtilityProcess = (child: UtilityProcess) => {
@@ -179,36 +190,3 @@ const handleMessagesFromFFmpegUtilityProcess = (child: UtilityProcess) => {
log.info("Ignoring unknown message from ffmpeg utility process", m);
});
};
/**
* The port exposed by _childFFmpeg (i.e., by the utility process running
* `ffmpeg-worker.ts`) provides an interface that conforms to
* {@link ElectronFFmpegWorker} (meant for use by the web layer), and in
* addition provides other "private" function meant for use by (the node layer).
*
* This interface lists the functions exposed for use by the node layer.
*/
export interface ElectronFFmpegWorkerNode {
ffmpegConvertToMP4: (
inputFilePath: string,
outputFilePath: string,
) => Promise<void>;
ffmpegGenerateHLSPlaylistAndSegments: (
inputFilePath: string,
outputPathPrefix: string,
) => Promise<FFmpegGenerateHLSPlaylistAndSegmentsResult | undefined>;
}
/**
* Return a handle to the already running ffmpeg utility process.
*
* This assumes that the web layer has already initiated the utility process by
* invoking `triggerCreateUtilityProcess("ffmpeg")`, otherwise this function
* will throw.
*/
export const electronFFmpegWorkerNodeIfRunning = () => {
const child = _childFFmpeg;
if (!child) throw new Error("ffmpeg utility process has not been started");
return child as unknown as ElectronFFmpegWorkerNode;
};

2
desktop/src/types/ipc.ts
View File

@@ -5,7 +5,7 @@
* See [Note: types.ts <-> preload.ts <-> ipc.ts]
*/
export type UtilityProcessType = "ml" | "ffmpeg";
export type UtilityProcessType = "ml";
export interface AppUpdate {
autoUpdatable: boolean;

85
web/packages/base/types/ipc.ts
View File

@@ -334,6 +334,44 @@ export interface Electron {
maxSize: number,
) => Promise<Uint8Array>;
// - FFmpeg
/**
* Execute a FFmpeg {@link command} on the given
* {@link dataOrPathOrZipItem}.
*
* This executes the command using a FFmpeg executable we bundle with our
* desktop app. We also have a Wasm FFmpeg implementation that we use when
* running on the web, which has a sibling function with the same
* parameters. See [Note:FFmpeg in Electron].
*
* @param command An array of strings, each representing one positional
* parameter in the command to execute. Placeholders for the input, output
* and ffmpeg's own path are replaced before executing the command
* (respectively {@link inputPathPlaceholder},
* {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}).
*
* @param dataOrPathOrZipItem The bytes of the input file, or the path to
* the input file on the user's local disk, or the path to a zip file on the
* user's disk and the name of an entry in it. In all three cases, the data
* gets serialized to a temporary file, and then that path gets substituted
* in the FFmpeg {@link command} in lieu of {@link inputPathPlaceholder}.
*
* @param outputFileExtension The extension (without the dot, e.g. "jpeg")
* to use for the output file that we ask FFmpeg to create in
* {@param command}. While this file will eventually get deleted, and we'll
* just return its contents, for some FFmpeg command the extension matters
* (e.g. conversion to a JPEG fails if the extension is arbitrary).
*
* @returns The contents of the output file produced by the ffmpeg command
* (specified as {@link outputPathPlaceholder} in {@link command}).
*/
ffmpegExec: (
command: FFmpegCommand,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
) => Promise<Uint8Array>;
// - Utility process
/**
@@ -349,8 +387,6 @@ export interface Electron {
* value of {@link type}. Thus, att the other end of that port will be an
* object that conforms to:
*
* - {@link ElectronFFmpegWorker} interface, when type is "ffmpeg".
*
* - {@link ElectronMLWorker} interface, when type is "ml".
*
* For more details about the IPC flow, see: [Note: ML IPC].
@@ -562,50 +598,7 @@ export interface Electron {
clearPendingUploads: () => Promise<void>;
}
export type UtilityProcessType = "ffmpeg" | "ml";
/**
* The shape of the object exposed by the Node.js utility process listening on
* the other side message port that the web layer obtains by doing
* {@link triggerCreateUtilityProcess} with type "ffmpeg".
*/
export interface ElectronFFmpegWorker {
/**
* Execute a FFmpeg {@link command} on the given
* {@link dataOrPathOrZipItem}.
*
* This executes the command using a FFmpeg executable we bundle with our
* desktop app. We also have a Wasm FFmpeg implementation that we use when
* running on the web, which has a sibling function with the same
* parameters. See [Note:FFmpeg in Electron].
*
* @param command An array of strings, each representing one positional
* parameter in the command to execute. Placeholders for the input, output
* and ffmpeg's own path are replaced before executing the command
* (respectively {@link inputPathPlaceholder},
* {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}).
*
* @param dataOrPathOrZipItem The bytes of the input file, or the path to
* the input file on the user's local disk, or the path to a zip file on the
* user's disk and the name of an entry in it. In all three cases, the data
* gets serialized to a temporary file, and then that path gets substituted
* in the FFmpeg {@link command} in lieu of {@link inputPathPlaceholder}.
*
* @param outputFileExtension The extension (without the dot, e.g. "jpeg")
* to use for the output file that we ask FFmpeg to create in
* {@param command}. While this file will eventually get deleted, and we'll
* just return its contents, for some FFmpeg command the extension matters
* (e.g. conversion to a JPEG fails if the extension is arbitrary).
*
* @returns The contents of the output file produced by the ffmpeg command
* (specified as {@link outputPathPlaceholder} in {@link command}).
*/
ffmpegExec: (
command: FFmpegCommand,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
) => Promise<Uint8Array>;
}
export type UtilityProcessType = "ml";
/**
* The shape of the object exposed by the Node.js utility process listening on

42
web/packages/gallery/services/ffmpeg/index.ts
View File

@@ -1,6 +1,6 @@
import { ensureElectron } from "ente-base/electron";
import log from "ente-base/log";
import type { Electron, ElectronFFmpegWorker } from "ente-base/types/ipc";
import type { Electron } from "ente-base/types/ipc";
import {
toDataOrPathOrZipEntry,
type FileSystemUploadItem,
@@ -16,7 +16,6 @@ import {
type ParsedMetadata,
} from "ente-media/file-metadata";
import { settingsSnapshot } from "ente-new/photos/services/settings";
import { createUtilityProcess } from "../../utils/native-worker";
import {
ffmpegPathPlaceholder,
inputPathPlaceholder,
@@ -24,21 +23,6 @@ import {
} from "./constants";
import { ffmpegExecWeb } from "./web";
let _electronFFmpegWorker: Promise<ElectronFFmpegWorker> | undefined;
/**
* Handle to the on-demand lazily created utility process in the Node.js layer
* that exposes an {@link ElectronFFmpegWorker} interface.
*/
export const getElectronFFmpegWorker = () =>
(_electronFFmpegWorker ??= createElectronFFmpegWorker());
const createElectronFFmpegWorker = () =>
createUtilityProcess(
ensureElectron(),
"ffmpeg",
) as unknown as Promise<ElectronFFmpegWorker>;
/**
* Generate a thumbnail for the given video using a Wasm FFmpeg running in a web
* worker.
@@ -92,15 +76,14 @@ const _generateVideoThumbnail = async (
* See also {@link generateVideoThumbnailNative}.
*/
export const generateVideoThumbnailNative = async (
electron: Electron,
fsUploadItem: FileSystemUploadItem,
) =>
getElectronFFmpegWorker().then((electronFW) =>
_generateVideoThumbnail((seekTime: number) =>
electronFW.ffmpegExec(
makeGenThumbnailCommand(seekTime),
toDataOrPathOrZipEntry(fsUploadItem),
"jpeg",
),
_generateVideoThumbnail((seekTime: number) =>
electron.ffmpegExec(
makeGenThumbnailCommand(seekTime),
toDataOrPathOrZipEntry(fsUploadItem),
"jpeg",
),
);
@@ -161,9 +144,11 @@ export const extractVideoMetadata = async (
return parseFFmpegExtractedMetadata(
uploadItem instanceof File
? await ffmpegExecWeb(command, uploadItem, "txt")
: await (
await getElectronFFmpegWorker()
).ffmpegExec(command, toDataOrPathOrZipEntry(uploadItem), "txt"),
: await ensureElectron().ffmpegExec(
command,
toDataOrPathOrZipEntry(uploadItem),
"txt",
),
);
};
@@ -312,8 +297,7 @@ export const convertToMP4 = async (blob: Blob): Promise<Blob | Uint8Array> => {
};
const convertToMP4Native = async (electron: Electron, blob: Blob) => {
const electronFFmpegWorker = await getElectronFFmpegWorker();
const token = await initiateConvertToMP4(electronFFmpegWorker, blob);
const token = await initiateConvertToMP4(electron, blob);
const mp4Blob = await readVideoStream(electron, token).then((res) =>
res.blob(),
);

2
web/packages/gallery/services/upload/thumbnail.ts
View File

@@ -200,7 +200,7 @@ export const generateThumbnailNative = async (
maxThumbnailDimension,
maxThumbnailSize,
)
: ffmpeg.generateVideoThumbnailNative(fsUploadItem);
: ffmpeg.generateVideoThumbnailNative(electron, fsUploadItem);
/**
* A fallback, black, thumbnail for use in cases where thumbnail generation

4
web/packages/gallery/services/video.ts
View File

@@ -23,7 +23,6 @@ import {
videoStreamDone,
} from "../utils/native-stream";
import { downloadManager } from "./download";
import { getElectronFFmpegWorker } from "./ffmpeg";
import {
fetchFileData,
fetchFilePreviewData,
@@ -554,7 +553,6 @@ const processQueueItem = async ({
timestampedUploadItem,
}: VideoProcessingQueueItem) => {
const electron = ensureElectron();
const electronFFmpegWorker = await getElectronFFmpegWorker();
log.debug(() => ["gen-hls", { file, timestampedUploadItem }]);
@@ -587,7 +585,7 @@ const processQueueItem = async ({
log.info(`Generate HLS for ${fileLogID(file)} | start`);
const res = await initiateGenerateHLS(
electronFFmpegWorker,
electron,
sourceVideo!,
objectUploadURL,
);

21
web/packages/gallery/utils/native-stream.ts
View File

@@ -6,12 +6,7 @@
* See: [Note: IPC streams].
*/
import type {
Electron,
ElectronFFmpegWorker,
ElectronMLWorker,
ZipItem,
} from "ente-base/types/ipc";
import type { Electron, ElectronMLWorker, ZipItem } from "ente-base/types/ipc";
import { z } from "zod";
import type { FileSystemUploadItem } from "../services/upload";
@@ -129,9 +124,8 @@ export const writeStream = async (
*
* This is a variant of {@link writeStream} tailored for the conversion to MP4.
*
* @param _ An {@link ElectronFFmpegWorker} instance, witness to the fact that
* we're running in the context of the desktop app, and that an ffmpeg utility
* process has been initialized. It is otherwise not used.
* @param _ An {@link Electron} instance, witness to the fact that we're running
* in the context of the desktop app. It is otherwise not used.
*
* @param video A {@link Blob} containing the video to convert.
*
@@ -142,7 +136,7 @@ export const writeStream = async (
* See: [Note: Convert to MP4].
*/
export const initiateConvertToMP4 = async (
_: ElectronFFmpegWorker,
_: Electron,
video: Blob,
): Promise<string> => {
const url = "stream://video?op=convert-to-mp4";
@@ -178,9 +172,8 @@ export type GenerateHLSResult = z.infer<typeof GenerateHLSResult>;
* is similar to {@link initiateConvertToMP4}, but also supports streaming
* {@link FileSystemUploadItem}s and {@link ReadableStream}s.
*
* @param _ An {@link ElectronFFmpegWorker} instance, witness to the fact that
* we're running in the context of the desktop app, and that an ffmpeg utility
* process has been initialized. It is otherwise not used.
* @param _ An {@link Electron} instance, witness to the fact that we're running
* in the context of the desktop app. It is otherwise not used.
*
* @param video The video to convert.
*
@@ -205,7 +198,7 @@ export type GenerateHLSResult = z.infer<typeof GenerateHLSResult>;
* See: [Note: Preview variant of videos].
*/
export const initiateGenerateHLS = async (
_: ElectronFFmpegWorker,
_: Electron,
video: FileSystemUploadItem | ReadableStream,
objectUploadURL: string,
): Promise<GenerateHLSResult | undefined> => {