From c86be54ac18024f8763d5a461cdcb9f4d5b53bf6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 6 Jul 2024 16:04:32 +0530 Subject: [PATCH 1/2] [desktop] Handle jp2 and simplify --- web/apps/cast/src/services/render.ts | 6 +- .../src/components/PhotoViewer/index.tsx | 19 ++-- web/packages/media/formats.ts | 29 ++++--- web/packages/new/photos/utils/file.ts | 86 ++++++++----------- 4 files changed, 66 insertions(+), 74 deletions(-) diff --git a/web/apps/cast/src/services/render.ts b/web/apps/cast/src/services/render.ts index acacdc88d2..7c9ea8e2c3 100644 --- a/web/apps/cast/src/services/render.ts +++ b/web/apps/cast/src/services/render.ts @@ -6,7 +6,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { FILE_TYPE } from "@/media/file-type"; -import { isHEICExtension, isNonWebImageFileExtension } from "@/media/formats"; +import { isHEICExtension, needsJPEGConversion } from "@/media/formats"; import { heicToJPEG } from "@/media/heic-convert"; import { decodeLivePhoto } from "@/media/live-photo"; import type { @@ -258,8 +258,8 @@ const isFileEligible = (file: EnteFile) => { // extension. To detect the actual type, we need to sniff the MIME type, but // that requires downloading and decrypting the file first. const [, extension] = nameAndExtension(file.metadata.title); - if (extension && isNonWebImageFileExtension(extension)) { - // Of the known non-web types, we support HEIC. + if (extension && needsJPEGConversion(extension)) { + // On the web, we only support HEIC conversion. return isHEICExtension(extension); } diff --git a/web/apps/photos/src/components/PhotoViewer/index.tsx b/web/apps/photos/src/components/PhotoViewer/index.tsx index 56ad5475ec..9d29c4a3b0 100644 --- a/web/apps/photos/src/components/PhotoViewer/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/index.tsx @@ -14,11 +14,11 @@ import { } from "utils/file"; import { FILE_TYPE } from "@/media/file-type"; -import { isNonWebImageFileExtension } from "@/media/formats"; +import { isHEICExtension, needsJPEGConversion } from "@/media/formats"; import downloadManager from "@/new/photos/services/download"; import type { LoadedLivePhotoSourceURL } from "@/new/photos/types/file"; import { detectFileTypeInfo } from "@/new/photos/utils/detect-type"; -import { isNativeConvertibleToJPEG } from "@/new/photos/utils/file"; +import { isDesktop } from "@/next/app"; import { lowercaseExtension } from "@/next/file"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; @@ -350,11 +350,16 @@ function PhotoViewer(props: Iprops) { function updateShowEditButton(file: EnteFile) { const extension = lowercaseExtension(file.metadata.title); - const isSupported = - !isNonWebImageFileExtension(extension) || - // TODO: This condition doesn't sound correct when running in the - // web app? - isNativeConvertibleToJPEG(extension); + // Assume it is supported. + let isSupported = true; + if (needsJPEGConversion(extension)) { + // See if the file is on the whitelist of extensions that we know + // will not be directly renderable. + if (!isDesktop) { + // On the web, we only support HEIC conversion. + isSupported = isHEICExtension(extension); + } + } setShowEditButton( file.metadata.fileType === FILE_TYPE.IMAGE && isSupported, ); diff --git a/web/packages/media/formats.ts b/web/packages/media/formats.ts index 1316b654f4..c2a98ca635 100644 --- a/web/packages/media/formats.ts +++ b/web/packages/media/formats.ts @@ -1,29 +1,30 @@ /** - * Image file extensions that we know the browser is unlikely to have native - * support for. + * List used by {@link needsJPEGConversion}. */ -const nonWebImageFileExtensions = [ - "heic", - "rw2", - "tiff", +const needsJPEGConversionExtensions = [ "arw", - "cr3", "cr2", - "raf", + "cr3", + "dng", + "heic", + "jp2", "nef", "psd", - "dng", + "rw2", "tif", + "tiff", ]; /** * Return `true` if {@link extension} is from amongst a known set of image file - * extensions that we know that the browser is unlikely to have native support - * for. If we want to display such files in the browser, we'll need to convert - * them to some other format first. + * extensions that (a) we know that the browser is unlikely to support, and (b) + * which we should be able to convert to JPEG when running in our desktop app. + * + * These two are independent constraints, but we only return true if we satisfy + * both of them instead of having two disjoint lists. */ -export const isNonWebImageFileExtension = (extension: string) => - nonWebImageFileExtensions.includes(extension.toLowerCase()); +export const needsJPEGConversion = (extension: string) => + needsJPEGConversionExtensions.includes(extension.toLowerCase()); /** * Return `true` if {@link extension} in for an HEIC-like file. diff --git a/web/packages/new/photos/utils/file.ts b/web/packages/new/photos/utils/file.ts index 7c4eaad36f..282737ef5b 100644 --- a/web/packages/new/photos/utils/file.ts +++ b/web/packages/new/photos/utils/file.ts @@ -1,4 +1,4 @@ -import { isNonWebImageFileExtension } from "@/media/formats"; +import { needsJPEGConversion } from "@/media/formats"; import { heicToJPEG } from "@/media/heic-convert"; import { isDesktop } from "@/next/app"; import log from "@/next/log"; @@ -104,9 +104,37 @@ export const renderableImageBlob = async ( ); const { extension } = fileTypeInfo; - if (!isNonWebImageFileExtension(extension)) { + if (needsJPEGConversion(extension)) { + const available = !moduleState.isNativeJPEGConversionNotAvailable; + if (isDesktop && available) { + // If we're running in our desktop app, see if our Node.js layer + // can convert this into a JPEG using native tools. + try { + return await nativeConvertToJPEG(imageBlob); + } catch (e) { + if ( + e instanceof Error && + e.message.endsWith(CustomErrorMessage.NotAvailable) + ) { + moduleState.isNativeJPEGConversionNotAvailable = true; + } else { + log.error("Native conversion to JPEG failed", e); + } + } + } + + if (extension == "heic" || extension == "heif") { + // If the previous step failed, or if native JPEG conversion is + // not available on this platform, for HEIC/HEIF files we can + // fallback to our web HEIC converter. + return await heicToJPEG(imageBlob); + } + + return undefined; + } else { // Either it is something that the browser already knows how to - // render, or something we don't even about yet. + // render, or a file extension that we haven't specifically + // whitelisted for conversion. const mimeType = fileTypeInfo.mimeType; if (!mimeType) { log.info( @@ -118,31 +146,6 @@ export const renderableImageBlob = async ( return new Blob([imageBlob], { type: mimeType }); } } - - const available = !moduleState.isNativeJPEGConversionNotAvailable; - if (isDesktop && available && isNativeConvertibleToJPEG(extension)) { - // If we're running in our desktop app, see if our Node.js layer can - // convert this into a JPEG using native tools for us. - try { - return await nativeConvertToJPEG(imageBlob); - } catch (e) { - if ( - e instanceof Error && - e.message.endsWith(CustomErrorMessage.NotAvailable) - ) { - moduleState.isNativeJPEGConversionNotAvailable = true; - } else { - log.error("Native conversion to JPEG failed", e); - } - } - } - - if (extension == "heic" || extension == "heif") { - // For HEIC/HEIF files we can use our web HEIC converter. - return await heicToJPEG(imageBlob); - } - - return undefined; } catch (e) { log.error(`Failed to get renderable image for ${fileName}`, e); return undefined; @@ -150,29 +153,12 @@ export const renderableImageBlob = async ( }; /** - * File extensions which our native JPEG conversion code should be able to - * convert to a renderable image. + * Convert {@link imageBlob} to a JPEG blob. + * + * The presumption is that method used by our desktop app for converting to JPEG + * should be able to handle files with all extensions for which + * {@link needsJPEGConversion} returns true. */ -const convertibleToJPEGExtensions = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "nef", - "psd", - "dng", - "tif", -]; - -/** - * Return true if {@link extension} is amongst the file extensions which we - * expect our native JPEG conversion to be able to process. - */ -export const isNativeConvertibleToJPEG = (extension: string) => - convertibleToJPEGExtensions.includes(extension.toLowerCase()); - const nativeConvertToJPEG = async (imageBlob: Blob) => { const startTime = Date.now(); const imageData = new Uint8Array(await imageBlob.arrayBuffer()); From d0f585fc97b68f703d43f7e3cb1a7f26e816d750 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 6 Jul 2024 16:57:46 +0530 Subject: [PATCH 2/2] Special case jp2 rendering --- web/packages/media/formats.ts | 9 +++++++ web/packages/new/photos/utils/file.ts | 34 +++++++++++++++------------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/web/packages/media/formats.ts b/web/packages/media/formats.ts index c2a98ca635..bc1bde9729 100644 --- a/web/packages/media/formats.ts +++ b/web/packages/media/formats.ts @@ -26,6 +26,15 @@ const needsJPEGConversionExtensions = [ export const needsJPEGConversion = (extension: string) => needsJPEGConversionExtensions.includes(extension.toLowerCase()); +/** + * Return true if {@link extension} _might_ be supported by the user's browser. + * + * For example, JPEG 2000 (jp2) is supported by Safari, but not by Chrome or + * Firefox, and this function will return true for `jp2`. + */ +export const hasPartialBrowserSupport = (extension: string) => + ["jp2"].includes(extension.toLowerCase()); + /** * Return `true` if {@link extension} in for an HEIC-like file. */ diff --git a/web/packages/new/photos/utils/file.ts b/web/packages/new/photos/utils/file.ts index 282737ef5b..8e878f9e8c 100644 --- a/web/packages/new/photos/utils/file.ts +++ b/web/packages/new/photos/utils/file.ts @@ -1,4 +1,4 @@ -import { needsJPEGConversion } from "@/media/formats"; +import { hasPartialBrowserSupport, needsJPEGConversion } from "@/media/formats"; import { heicToJPEG } from "@/media/heic-convert"; import { isDesktop } from "@/next/app"; import log from "@/next/log"; @@ -130,21 +130,25 @@ export const renderableImageBlob = async ( return await heicToJPEG(imageBlob); } - return undefined; + // Continue if it might be possibly supported in some browsers, + // otherwise bail out. + if (!hasPartialBrowserSupport(extension)) return undefined; + } + + // Either it is something that the browser already knows how to render + // (e.g. JPEG/PNG), or is a file extension that might be supported in + // some browsers (e.g. JPEG 2000), or a file extension that we haven't + // specifically whitelisted for conversion (any arbitrary extension not + // part of `needsJPEGConversion`). + // + // Give it to the browser, attaching the mime type if possible. + + const mimeType = fileTypeInfo.mimeType; + if (!mimeType) { + log.info("Trying to render a file without a MIME type", fileName); + return imageBlob; } else { - // Either it is something that the browser already knows how to - // render, or a file extension that we haven't specifically - // whitelisted for conversion. - const mimeType = fileTypeInfo.mimeType; - if (!mimeType) { - log.info( - "Trying to render a file without a MIME type", - fileName, - ); - return imageBlob; - } else { - return new Blob([imageBlob], { type: mimeType }); - } + return new Blob([imageBlob], { type: mimeType }); } } catch (e) { log.error(`Failed to get renderable image for ${fileName}`, e);