TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] Improve JPEG 2000 handling (#2383)

Browse Source 
- Let supporting browsers (e.g. Safari) upload them.
- Let them be indexed by converting to JPEG.
This commit is contained in:
Manav Rathi
2024-07-06 17:05:03 +05:30
committed by GitHub
parent d24c116d14 d0f585fc97
commit 2f2d988eaa
4 changed files with 86 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
web/apps/cast/src/services/render.ts
View File

@@ -6,7 +6,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { FILE_TYPE } from "@/media/file-type";
import { isHEICExtension, isNonWebImageFileExtension } from "@/media/formats";
import { isHEICExtension, needsJPEGConversion } from "@/media/formats";
import { heicToJPEG } from "@/media/heic-convert";
import { decodeLivePhoto } from "@/media/live-photo";
import type {
@@ -258,8 +258,8 @@ const isFileEligible = (file: EnteFile) => {
// extension. To detect the actual type, we need to sniff the MIME type, but
// that requires downloading and decrypting the file first.
const [, extension] = nameAndExtension(file.metadata.title);
if (extension && isNonWebImageFileExtension(extension)) {
// Of the known non-web types, we support HEIC.
if (extension && needsJPEGConversion(extension)) {
// On the web, we only support HEIC conversion.
return isHEICExtension(extension);
}

19
web/apps/photos/src/components/PhotoViewer/index.tsx
View File

@@ -14,11 +14,11 @@ import {
} from "utils/file";
import { FILE_TYPE } from "@/media/file-type";
import { isNonWebImageFileExtension } from "@/media/formats";
import { isHEICExtension, needsJPEGConversion } from "@/media/formats";
import downloadManager from "@/new/photos/services/download";
import type { LoadedLivePhotoSourceURL } from "@/new/photos/types/file";
import { detectFileTypeInfo } from "@/new/photos/utils/detect-type";
import { isNativeConvertibleToJPEG } from "@/new/photos/utils/file";
import { isDesktop } from "@/next/app";
import { lowercaseExtension } from "@/next/file";
import { FlexWrapper } from "@ente/shared/components/Container";
import EnteSpinner from "@ente/shared/components/EnteSpinner";
@@ -350,11 +350,16 @@ function PhotoViewer(props: Iprops) {
function updateShowEditButton(file: EnteFile) {
const extension = lowercaseExtension(file.metadata.title);
const isSupported =
!isNonWebImageFileExtension(extension) ||
// TODO: This condition doesn't sound correct when running in the
// web app?
isNativeConvertibleToJPEG(extension);
// Assume it is supported.
let isSupported = true;
if (needsJPEGConversion(extension)) {
// See if the file is on the whitelist of extensions that we know
// will not be directly renderable.
if (!isDesktop) {
// On the web, we only support HEIC conversion.
isSupported = isHEICExtension(extension);
}
}
setShowEditButton(
file.metadata.fileType === FILE_TYPE.IMAGE && isSupported,
);

38
web/packages/media/formats.ts
View File

@@ -1,29 +1,39 @@
/**
* Image file extensions that we know the browser is unlikely to have native
* support for.
* List used by {@link needsJPEGConversion}.
*/
const nonWebImageFileExtensions = [
"heic",
"rw2",
"tiff",
const needsJPEGConversionExtensions = [
"arw",
"cr3",
"cr2",
"raf",
"cr3",
"dng",
"heic",
"jp2",
"nef",
"psd",
"dng",
"rw2",
"tif",
"tiff",
];
/**
* Return `true` if {@link extension} is from amongst a known set of image file
* extensions that we know that the browser is unlikely to have native support
* for. If we want to display such files in the browser, we'll need to convert
* them to some other format first.
* extensions that (a) we know that the browser is unlikely to support, and (b)
* which we should be able to convert to JPEG when running in our desktop app.
*
* These two are independent constraints, but we only return true if we satisfy
* both of them instead of having two disjoint lists.
*/
export const isNonWebImageFileExtension = (extension: string) =>
nonWebImageFileExtensions.includes(extension.toLowerCase());
export const needsJPEGConversion = (extension: string) =>
needsJPEGConversionExtensions.includes(extension.toLowerCase());
/**
* Return true if {@link extension} _might_ be supported by the user's browser.
*
* For example, JPEG 2000 (jp2) is supported by Safari, but not by Chrome or
* Firefox, and this function will return true for `jp2`.
*/
export const hasPartialBrowserSupport = (extension: string) =>
["jp2"].includes(extension.toLowerCase());
/**
* Return `true` if {@link extension} in for an HEIC-like file.

104
web/packages/new/photos/utils/file.ts
View File

@@ -1,4 +1,4 @@
import { isNonWebImageFileExtension } from "@/media/formats";
import { hasPartialBrowserSupport, needsJPEGConversion } from "@/media/formats";
import { heicToJPEG } from "@/media/heic-convert";
import { isDesktop } from "@/next/app";
import log from "@/next/log";
@@ -104,45 +104,52 @@ export const renderableImageBlob = async (
);
const { extension } = fileTypeInfo;
if (!isNonWebImageFileExtension(extension)) {
// Either it is something that the browser already knows how to
// render, or something we don't even about yet.
const mimeType = fileTypeInfo.mimeType;
if (!mimeType) {
log.info(
"Trying to render a file without a MIME type",
fileName,
);
return imageBlob;
} else {
return new Blob([imageBlob], { type: mimeType });
}
}
const available = !moduleState.isNativeJPEGConversionNotAvailable;
if (isDesktop && available && isNativeConvertibleToJPEG(extension)) {
// If we're running in our desktop app, see if our Node.js layer can
// convert this into a JPEG using native tools for us.
try {
return await nativeConvertToJPEG(imageBlob);
} catch (e) {
if (
e instanceof Error &&
e.message.endsWith(CustomErrorMessage.NotAvailable)
) {
moduleState.isNativeJPEGConversionNotAvailable = true;
} else {
log.error("Native conversion to JPEG failed", e);
if (needsJPEGConversion(extension)) {
const available = !moduleState.isNativeJPEGConversionNotAvailable;
if (isDesktop && available) {
// If we're running in our desktop app, see if our Node.js layer
// can convert this into a JPEG using native tools.
try {
return await nativeConvertToJPEG(imageBlob);
} catch (e) {
if (
e instanceof Error &&
e.message.endsWith(CustomErrorMessage.NotAvailable)
) {
moduleState.isNativeJPEGConversionNotAvailable = true;
} else {
log.error("Native conversion to JPEG failed", e);
}
}
}
if (extension == "heic" || extension == "heif") {
// If the previous step failed, or if native JPEG conversion is
// not available on this platform, for HEIC/HEIF files we can
// fallback to our web HEIC converter.
return await heicToJPEG(imageBlob);
}
// Continue if it might be possibly supported in some browsers,
// otherwise bail out.
if (!hasPartialBrowserSupport(extension)) return undefined;
}
if (extension == "heic" || extension == "heif") {
// For HEIC/HEIF files we can use our web HEIC converter.
return await heicToJPEG(imageBlob);
}
// Either it is something that the browser already knows how to render
// (e.g. JPEG/PNG), or is a file extension that might be supported in
// some browsers (e.g. JPEG 2000), or a file extension that we haven't
// specifically whitelisted for conversion (any arbitrary extension not
// part of `needsJPEGConversion`).
//
// Give it to the browser, attaching the mime type if possible.
return undefined;
const mimeType = fileTypeInfo.mimeType;
if (!mimeType) {
log.info("Trying to render a file without a MIME type", fileName);
return imageBlob;
} else {
return new Blob([imageBlob], { type: mimeType });
}
} catch (e) {
log.error(`Failed to get renderable image for ${fileName}`, e);
return undefined;
@@ -150,29 +157,12 @@ export const renderableImageBlob = async (
};
/**
* File extensions which our native JPEG conversion code should be able to
* convert to a renderable image.
* Convert {@link imageBlob} to a JPEG blob.
*
* The presumption is that method used by our desktop app for converting to JPEG
* should be able to handle files with all extensions for which
* {@link needsJPEGConversion} returns true.
*/
const convertibleToJPEGExtensions = [
"heic",
"rw2",
"tiff",
"arw",
"cr3",
"cr2",
"nef",
"psd",
"dng",
"tif",
];
/**
* Return true if {@link extension} is amongst the file extensions which we
* expect our native JPEG conversion to be able to process.
*/
export const isNativeConvertibleToJPEG = (extension: string) =>
convertibleToJPEGExtensions.includes(extension.toLowerCase());
const nativeConvertToJPEG = async (imageBlob: Blob) => {
const startTime = Date.now();
const imageData = new Uint8Array(await imageBlob.arrayBuffer());