TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move

Browse Source
This commit is contained in:
Manav Rathi
2024-11-22 14:32:53 +05:30
parent 60b0be15ad
commit 6fe994a65c
2 changed files with 109 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
web/packages/gallery/utils/convert.ts
View File

@@ -4,6 +4,7 @@ import { CustomErrorMessage } from "@/base/types/ipc";
import { workerBridge } from "@/base/worker/worker-bridge";
import { isHEICExtension, needsJPEGConversion } from "@/media/formats";
import { heicToJPEG } from "@/media/heic-convert";
import { convertToMP4 } from "../services/ffmpeg";
import { detectFileTypeInfo } from "./detect-type";
/**
@@ -16,8 +17,8 @@ import { detectFileTypeInfo } from "./detect-type";
let _isNativeJPEGConversionAvailable = true;
/**
* Return a new {@link Blob} containing data in a format that the browser
* (likely) knows how to render (in an img tag, or on the canvas).
* Return a new {@link Blob} containing an image's data in a format that the
* browser (likely) knows how to render (in an img tag, or on the canvas).
*
* The type of the returned blob is set, whenever possible, to the MIME type of
* the data that we're dealing with.
@@ -55,9 +56,9 @@ export const renderableImageBlob = async (
const fileTypeInfo = await detectFileTypeInfo(file);
const { extension, mimeType } = fileTypeInfo;
log.debug(() => ["Get renderable blob", { fileName, ...fileTypeInfo }]);
if (needsJPEGConversion(extension)) {
log.debug(() => [`Converting ${fileName} to JPEG`, fileTypeInfo]);
// If we're running in our desktop app, see if our Node.js layer can
// convert this into a JPEG using native tools.
@@ -95,7 +96,7 @@ export const renderableImageBlob = async (
if (!mimeType) {
log.info(
"Attempting to get renderable blob for a file without a MIME type",
"Attempting to convert a file without a MIME type",
fileName,
);
return imageBlob;
@@ -103,10 +104,7 @@ export const renderableImageBlob = async (
return new Blob([imageBlob], { type: mimeType });
}
} catch (e) {
log.error(
`Failed to get renderable blob for ${fileName}, will fallback to the original`,
e,
);
log.error(`Failed to convert ${fileName}, will use the original`, e);
return imageBlob;
}
};
@@ -131,3 +129,96 @@ const nativeConvertToJPEG = async (imageBlob: Blob) => {
log.debug(() => `Native JPEG conversion took ${Date.now() - startTime} ms`);
return new Blob([jpegData], { type: "image/jpeg" });
};
/**
* Return a new {@link Blob} containing a video's data in a format that the
* browser (likely) knows how to play back (using an video tag).
*
* Unlike {@link renderableImageBlob}, this uses a much simpler flowchart:
*
* - If the browser thinks it can play the video, then return the original blob
* back.
*
* - Otherwise try to convert using FFmpeg. This conversion always happens on
* the desktop app, but in the browser the conversion only happens for short
* videos since the WASM FFmpeg implementation is much slower. There is also a
* flag to force this conversion regardless.
*/
export const playableVideoBlob = async (
fileName: string,
videoBlob: Blob,
forceConvert: boolean,
) => {
const converted = async () => {
try {
log.info(`Converting ${fileName} to mp4`);
const convertedBlob = await convertToMP4(videoBlob);
return new Blob([convertedBlob], { type: "video/mp4" });
} catch (e) {
log.error(`Video conversion failed for ${fileName}`, e);
return null;
}
};
// If we've been asked to force convert, do it regardless of anything else.
if (forceConvert) return converted();
const isPlayable = await isPlaybackPossible(URL.createObjectURL(videoBlob));
if (isPlayable) return videoBlob;
// The browser doesn't think it can play this video, try transcoding.
if (isDesktop) {
return converted();
} else {
// Don't try to transcode on the web if the file is too big.
if (videoBlob.size > 100 * 1024 * 1024 /* 100 MB, arbitrary */) {
return null;
} else {
return converted();
}
}
};
/**
* Try to see if the browser thinks it can play the video pointed to by the
* given {@link url} by creating a <video>
* element and initiating playback.
*
* Note that this can sometimes cause false positives if the browser can play
* some of the streams in the video, but not all. For example, the browser may
* be able to play back the video stream, but not the audio stream (say due to
* some codec issue): in such cases this function will return true, causing us
* to skip conversion, but when the user actually plays the video there will be
* no sound.
*
* As an escape hatch, we provide a force convert button in the UI for such
* cases, which'll cause the {@link forceConvert} flag in our caller function to
* be set (and force a conversion)
*/
const isPlaybackPossible = async (url: string) =>
new Promise((resolve) => {
const t = setTimeout(() => {
video.remove();
resolve(false);
}, 1000);
const video = document.createElement("video");
video.addEventListener("canplay", () => {
clearTimeout(t);
// Clean up the video element.
video.remove();
// Check for duration > 0 to make sure it is not a broken video.
if (video.duration > 0) {
resolve(true);
} else {
resolve(false);
}
});
video.addEventListener("error", () => {
clearTimeout(t);
video.remove();
resolve(false);
});
video.src = url;
});

83
web/packages/new/photos/services/download.ts
View File

@@ -1,7 +1,6 @@
// TODO: Remove this override
/* eslint-disable @typescript-eslint/no-empty-function */
import { isDesktop } from "@/base/app";
import { blobCache, type BlobCache } from "@/base/blob-cache";
import {
decryptStreamBytes,
@@ -11,8 +10,10 @@ import {
} from "@/base/crypto";
import log from "@/base/log";
import { customAPIOrigin } from "@/base/origins";
import { convertToMP4 } from "@/gallery/services/ffmpeg";
import { renderableImageBlob } from "@/gallery/utils/convert";
import {
playableVideoBlob,
renderableImageBlob,
} from "@/gallery/utils/convert";
import { retryAsyncOperation } from "@/gallery/utils/retry-async";
import type { EnteFile, LivePhotoSourceURL, SourceURLs } from "@/media/file";
import { FileType } from "@/media/file-type";
@@ -452,12 +453,10 @@ async function getRenderableFileURL(
let type: SourceURLs["type"] = "normal";
let mimeType: string | undefined;
const fileName = file.metadata.title;
switch (file.metadata.fileType) {
case FileType.image: {
const convertedBlob = await renderableImageBlob(
file.metadata.title,
fileBlob,
);
const convertedBlob = await renderableImageBlob(fileName, fileBlob);
const convertedURL = existingOrNewObjectURL(convertedBlob);
url = convertedURL;
isOriginal = convertedURL === originalFileURL;
@@ -473,8 +472,8 @@ async function getRenderableFileURL(
break;
}
case FileType.video: {
const convertedBlob = await getPlayableVideo(
file.metadata.title,
const convertedBlob = await playableVideoBlob(
fileName,
fileBlob,
forceConvertVideos,
);
@@ -518,7 +517,7 @@ async function getRenderableLivePhotoURL(
const getRenderableLivePhotoVideoURL = async () => {
try {
const videoBlob = new Blob([livePhoto.videoData]);
const convertedVideoBlob = await getPlayableVideo(
const convertedVideoBlob = await playableVideoBlob(
livePhoto.videoFileName,
videoBlob,
false,
@@ -537,70 +536,6 @@ async function getRenderableLivePhotoURL(
};
}
async function getPlayableVideo(
videoNameTitle: string,
videoBlob: Blob,
forceConvert: boolean,
) {
const converted = async () => {
try {
log.info(`Converting video ${videoNameTitle} to mp4`);
const convertedVideoData = await convertToMP4(videoBlob);
return new Blob([convertedVideoData], { type: "video/mp4" });
} catch (e) {
log.error("Video conversion failed", e);
return null;
}
};
// If we've been asked to force convert, do it regardless of anything else.
if (forceConvert) return converted();
const isPlayable = await isPlaybackPossible(URL.createObjectURL(videoBlob));
if (isPlayable) return videoBlob;
// The browser doesn't think it can play this video, try transcoding.
if (isDesktop) {
return converted();
} else {
// Don't try to transcode on the web if the file is too big.
if (videoBlob.size > 100 * 1024 * 1024 /* 100 MB */) {
return null;
} else {
return converted();
}
}
}
const WAIT_FOR_VIDEO_PLAYBACK = 1 * 1000;
async function isPlaybackPossible(url: string): Promise<boolean> {
return await new Promise((resolve) => {
const t = setTimeout(() => {
resolve(false);
}, WAIT_FOR_VIDEO_PLAYBACK);
const video = document.createElement("video");
video.addEventListener("canplay", function () {
clearTimeout(t);
video.remove(); // Clean up the video element
// also check for duration > 0 to make sure it is not a broken video
if (video.duration > 0) {
resolve(true);
} else {
resolve(false);
}
});
video.addEventListener("error", function () {
clearTimeout(t);
video.remove();
resolve(false);
});
video.src = url;
});
}
class PhotosDownloadClient implements DownloadClient {
constructor(
private token: string,