diff --git a/web/apps/cast/src/services/render.ts b/web/apps/cast/src/services/render.ts index 40b45e7e8e..37af7201ce 100644 --- a/web/apps/cast/src/services/render.ts +++ b/web/apps/cast/src/services/render.ts @@ -1,11 +1,9 @@ import { FILE_TYPE } from "@/media/file-type"; import { isHEICExtension, isNonWebImageFileExtension } from "@/media/formats"; +import { heicToJPEG } from "@/media/heic-convert"; import { decodeLivePhoto } from "@/media/live-photo"; -import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert"; -import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker"; import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; -import type { ComlinkWorker } from "@/next/worker/comlink-worker"; import { shuffled } from "@/utils/array"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; @@ -24,12 +22,6 @@ import type { } from "types/file"; import { isChromecast } from "./chromecast"; -/** - * If we're using HEIC conversion, then this variable caches the comlink web - * worker we're using to perform the actual conversion. - */ -let heicWorker: ComlinkWorker | undefined; - /** * An async generator function that loops through all the files in the * collection, returning renderable image URLs to each that can be displayed in @@ -270,12 +262,6 @@ const isImageOrLivePhoto = (file: EnteFile) => { return fileType == FILE_TYPE.IMAGE || fileType == FILE_TYPE.LIVE_PHOTO; }; -export const heicToJPEG = async (heicBlob: Blob) => { - let worker = heicWorker; - if (!worker) heicWorker = worker = createHEICConvertComlinkWorker(); - return await (await worker.remote).heicToJPEG(heicBlob); -}; - /** * Create and return a new data URL that can be used to show the given * {@link file} in our slideshow image viewer. diff --git a/web/apps/photos/src/services/heic-convert.ts b/web/apps/photos/src/services/heic-convert.ts index d2e05d9ec9..e69de29bb2 100644 --- a/web/apps/photos/src/services/heic-convert.ts +++ b/web/apps/photos/src/services/heic-convert.ts @@ -1,100 +0,0 @@ -import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert"; -import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker"; -import log from "@/next/log"; -import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { CustomError } from "@ente/shared/error"; -import { retryAsyncFunction } from "@ente/shared/utils"; -import QueueProcessor from "@ente/shared/utils/queueProcessor"; - -/** - * Convert a HEIC image to a JPEG. - * - * Behind the scenes, it uses a web worker pool to do the conversion using a - * WASM HEIC conversion package. - * - * @param heicBlob The HEIC blob to convert. - * @returns The JPEG blob. - */ -export const heicToJPEG = (heicBlob: Blob) => converter.convert(heicBlob); - -const WORKER_POOL_SIZE = 2; -const WAIT_TIME_BEFORE_NEXT_ATTEMPT_IN_MICROSECONDS = [100, 100]; -const WAIT_TIME_IN_MICROSECONDS = 30 * 1000; -const BREATH_TIME_IN_MICROSECONDS = 1000; - -class HEICConverter { - private convertProcessor = new QueueProcessor(); - private workerPool: ComlinkWorker[] = []; - - private initIfNeeded() { - if (this.workerPool.length > 0) return; - this.workerPool = []; - for (let i = 0; i < WORKER_POOL_SIZE; i++) - this.workerPool.push(createHEICConvertComlinkWorker()); - } - - async convert(fileBlob: Blob): Promise { - this.initIfNeeded(); - - const response = this.convertProcessor.queueUpRequest(() => - retryAsyncFunction(async () => { - const convertWorker = this.workerPool.shift(); - const worker = await convertWorker.remote; - try { - const convertedHEIC = await new Promise( - (resolve, reject) => { - const main = async () => { - try { - const timeout = setTimeout(() => { - reject(Error("wait time exceeded")); - }, WAIT_TIME_IN_MICROSECONDS); - const startTime = Date.now(); - const convertedHEIC = - await worker.heicToJPEG(fileBlob); - const ms = Date.now() - startTime; - log.debug(() => `heic => jpeg (${ms} ms)`); - clearTimeout(timeout); - resolve(convertedHEIC); - } catch (e) { - reject(e); - } - }; - main(); - }, - ); - if (!convertedHEIC || convertedHEIC?.size === 0) { - log.error( - `Converted HEIC file is empty (original was ${fileBlob?.size} bytes)`, - ); - } - await new Promise((resolve) => { - setTimeout( - () => resolve(null), - BREATH_TIME_IN_MICROSECONDS, - ); - }); - this.workerPool.push(convertWorker); - return convertedHEIC; - } catch (e) { - log.error("HEIC conversion failed", e); - convertWorker.terminate(); - this.workerPool.push(createHEICConvertComlinkWorker()); - throw e; - } - }, WAIT_TIME_BEFORE_NEXT_ATTEMPT_IN_MICROSECONDS), - ); - - try { - return await response.promise; - } catch (e) { - if (e.message === CustomError.REQUEST_CANCELLED) { - // ignore - return null; - } - throw e; - } - } -} - -/** The singleton instance of {@link HEICConverter}. */ -const converter = new HEICConverter(); diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 10da88a650..91260131d3 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,11 +1,11 @@ import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type"; +import { heicToJPEG } from "@/media/heic-convert"; import { scaledImageDimensions } from "@/media/image"; import log from "@/next/log"; import { type Electron } from "@/next/types/ipc"; import { ensure } from "@/utils/ensure"; import { withTimeout } from "@/utils/promise"; import * as ffmpeg from "services/ffmpeg"; -import { heicToJPEG } from "services/heic-convert"; import { toDataOrPathOrZipEntry, type DesktopUploadItem } from "./types"; /** Maximum width or height of the generated thumbnail */ diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 2879bdd757..5e2c60cee2 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,5 +1,6 @@ import { FILE_TYPE } from "@/media/file-type"; import { isNonWebImageFileExtension } from "@/media/formats"; +import { heicToJPEG } from "@/media/heic-convert"; import { decodeLivePhoto } from "@/media/live-photo"; import { lowercaseExtension } from "@/next/file"; import log from "@/next/log"; @@ -22,7 +23,6 @@ import { updateFileMagicMetadata, updateFilePublicMagicMetadata, } from "services/fileService"; -import { heicToJPEG } from "services/heic-convert"; import { EncryptedEnteFile, EnteFile, diff --git a/web/packages/media/heic-convert.ts b/web/packages/media/heic-convert.ts new file mode 100644 index 0000000000..b323f014a7 --- /dev/null +++ b/web/packages/media/heic-convert.ts @@ -0,0 +1,36 @@ +import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { wait } from "@/utils/promise"; +import type { HEICConvertWorker } from "./heic-convert.worker"; + +/** + * Convert a HEIC image to a JPEG. + * + * Behind the scenes, it uses a web worker to do the conversion using a WASM + * HEIC conversion package. + * + * @param heicBlob The HEIC blob to convert. + * + * @returns The JPEG blob. + */ +export const heicToJPEG = async (heicBlob: Blob) => + worker() + .then((w) => w.heicToJPEG(heicBlob)) + // I'm told this library used to have big memory spikes, and adding pauses + // to get GC to run helped. + .then((res) => wait(10 /* ms */).then(() => res)); + +/** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ +let _comlinkWorker: ComlinkWorker | undefined; + +/** Lazily created, cached, instance of our web worker. */ +const worker = async () => { + let comlinkWorker = _comlinkWorker; + if (!comlinkWorker) _comlinkWorker = comlinkWorker = createComlinkWorker(); + return await comlinkWorker.remote; +}; + +const createComlinkWorker = () => + new ComlinkWorker( + "heic-convert-worker", + new Worker(new URL("heic-convert.worker.ts", import.meta.url)), + ); diff --git a/web/packages/media/worker/heic-convert.worker.ts b/web/packages/media/heic-convert.worker.ts similarity index 63% rename from web/packages/media/worker/heic-convert.worker.ts rename to web/packages/media/heic-convert.worker.ts index ffb5eb1582..5d8383c0a9 100644 --- a/web/packages/media/worker/heic-convert.worker.ts +++ b/web/packages/media/heic-convert.worker.ts @@ -1,20 +1,20 @@ import { expose } from "comlink"; import HeicConvert from "heic-convert"; -export class DedicatedHEICConvertWorker { +export class HEICConvertWorker { + /** + * Convert a HEIC file to a JPEG file. + * + * Both the input and output are blobs. + */ async heicToJPEG(heicBlob: Blob) { return heicToJPEG(heicBlob); } } -expose(DedicatedHEICConvertWorker); +expose(HEICConvertWorker); -/** - * Convert a HEIC file to a JPEG file. - * - * Both the input and output are blobs. - */ -export const heicToJPEG = async (heicBlob: Blob): Promise => { +const heicToJPEG = async (heicBlob: Blob): Promise => { const buffer = new Uint8Array(await heicBlob.arrayBuffer()); const result = await HeicConvert({ buffer, format: "JPEG" }); const convertedData = new Uint8Array(result); diff --git a/web/packages/media/worker/heic-convert.ts b/web/packages/media/worker/heic-convert.ts deleted file mode 100644 index 476eac00a3..0000000000 --- a/web/packages/media/worker/heic-convert.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import type { DedicatedHEICConvertWorker } from "./heic-convert.worker"; - -export const createHEICConvertWebWorker = () => - new Worker(new URL("heic-convert.worker.ts", import.meta.url)); - -export const createHEICConvertComlinkWorker = () => - new ComlinkWorker( - "heic-convert-worker", - createHEICConvertWebWorker(), - );