diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 9ea4d802fa..15d51530df 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -38,9 +38,9 @@ import { updateAndRestart, updateOnNextRestart, } from "./services/app-update"; -import { convertToJPEG, generateImageThumbnail } from "./services/convert"; import { ffmpegExec } from "./services/ffmpeg"; import { getDirFiles } from "./services/fs"; +import { convertToJPEG, generateImageThumbnail } from "./services/image"; import { clipImageEmbedding, clipTextEmbeddingIfAvailable, @@ -141,14 +141,18 @@ export const attachIPCHandlers = () => { // - Conversion - ipcMain.handle("convertToJPEG", (_, fileName, imageData) => - convertToJPEG(fileName, imageData), + ipcMain.handle("convertToJPEG", (_, imageData: Uint8Array) => + convertToJPEG(imageData), ); ipcMain.handle( "generateImageThumbnail", - (_, inputFile, maxDimension, maxSize) => - generateImageThumbnail(inputFile, maxDimension, maxSize), + ( + _, + dataOrPath: Uint8Array | string, + maxDimension: number, + maxSize: number, + ) => generateImageThumbnail(dataOrPath, maxDimension, maxSize), ); ipcMain.handle( @@ -156,10 +160,10 @@ export const attachIPCHandlers = () => { ( _, command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, - ) => ffmpegExec(command, inputDataOrPath, outputFileName, timeoutMS), + ) => ffmpegExec(command, dataOrPath, outputFileExtension, timeoutMS), ); // - ML diff --git a/desktop/src/main/services/convert.ts b/desktop/src/main/services/convert.ts deleted file mode 100644 index 7f38a86ea6..0000000000 --- a/desktop/src/main/services/convert.ts +++ /dev/null @@ -1,241 +0,0 @@ -/** @file Image conversions */ - -import { existsSync } from "fs"; -import fs from "node:fs/promises"; -import path from "path"; -import { CustomErrorMessage, ElectronFile } from "../../types/ipc"; -import log from "../log"; -import { writeStream } from "../stream"; -import { execAsync, isDev } from "../utils-electron"; -import { deleteTempFile, makeTempFilePath } from "../utils-temp"; - -export const convertToJPEG = async ( - fileName: string, - imageData: Uint8Array, -): Promise => { - const inputFilePath = await makeTempFilePath(fileName); - const outputFilePath = await makeTempFilePath("output.jpeg"); - - // Construct the command first, it may throw on NotAvailable on win32. - const command = convertToJPEGCommand(inputFilePath, outputFilePath); - - try { - await fs.writeFile(inputFilePath, imageData); - await execAsync(command); - return new Uint8Array(await fs.readFile(outputFilePath)); - } finally { - try { - deleteTempFile(outputFilePath); - deleteTempFile(inputFilePath); - } catch (e) { - log.error("Ignoring error when cleaning up temp files", e); - } - } -}; - -const convertToJPEGCommand = ( - inputFilePath: string, - outputFilePath: string, -) => { - switch (process.platform) { - case "darwin": - return [ - "sips", - "-s", - "format", - "jpeg", - inputFilePath, - "--out", - outputFilePath, - ]; - case "linux": - return [ - imageMagickPath(), - inputFilePath, - "-quality", - "100%", - outputFilePath, - ]; - default: // "win32" - throw new Error(CustomErrorMessage.NotAvailable); - } -}; - -/** Path to the Linux image-magick executable bundled with our app */ -const imageMagickPath = () => - path.join(isDev ? "build" : process.resourcesPath, "image-magick"); - -const IMAGE_MAGICK_PLACEHOLDER = "IMAGE_MAGICK"; -const MAX_DIMENSION_PLACEHOLDER = "MAX_DIMENSION"; -const SAMPLE_SIZE_PLACEHOLDER = "SAMPLE_SIZE"; -const INPUT_PATH_PLACEHOLDER = "INPUT"; -const OUTPUT_PATH_PLACEHOLDER = "OUTPUT"; -const QUALITY_PLACEHOLDER = "QUALITY"; - -const MAX_QUALITY = 70; -const MIN_QUALITY = 50; - -const SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ - "sips", - "-s", - "format", - "jpeg", - "-s", - "formatOptions", - QUALITY_PLACEHOLDER, - "-Z", - MAX_DIMENSION_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - "--out", - OUTPUT_PATH_PLACEHOLDER, -]; - -const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ - IMAGE_MAGICK_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - "-auto-orient", - "-define", - `jpeg:size=${SAMPLE_SIZE_PLACEHOLDER}x${SAMPLE_SIZE_PLACEHOLDER}`, - "-thumbnail", - `${MAX_DIMENSION_PLACEHOLDER}x${MAX_DIMENSION_PLACEHOLDER}>`, - "-unsharp", - "0x.5", - "-quality", - QUALITY_PLACEHOLDER, - OUTPUT_PATH_PLACEHOLDER, -]; - -export async function generateImageThumbnail( - inputFile: File | ElectronFile, - maxDimension: number, - maxSize: number, -): Promise { - let inputFilePath = null; - let createdTempInputFile = null; - try { - if (process.platform == "win32") - throw Error( - CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED, - ); - if (!existsSync(inputFile.path)) { - const tempFilePath = await makeTempFilePath(inputFile.name); - await writeStream(tempFilePath, await inputFile.stream()); - inputFilePath = tempFilePath; - createdTempInputFile = true; - } else { - inputFilePath = inputFile.path; - } - const thumbnail = await generateImageThumbnail_( - inputFilePath, - maxDimension, - maxSize, - ); - return thumbnail; - } finally { - if (createdTempInputFile) { - try { - await deleteTempFile(inputFilePath); - } catch (e) { - log.error(`Failed to deleteTempFile ${inputFilePath}`, e); - } - } - } -} - -async function generateImageThumbnail_( - inputFilePath: string, - width: number, - maxSize: number, -): Promise { - let tempOutputFilePath: string; - let quality = MAX_QUALITY; - try { - tempOutputFilePath = await makeTempFilePath("thumb.jpeg"); - let thumbnail: Uint8Array; - do { - await execAsync( - constructThumbnailGenerationCommand( - inputFilePath, - tempOutputFilePath, - width, - quality, - ), - ); - thumbnail = new Uint8Array(await fs.readFile(tempOutputFilePath)); - quality -= 10; - } while (thumbnail.length > maxSize && quality > MIN_QUALITY); - return thumbnail; - } catch (e) { - log.error("Failed to generate image thumbnail", e); - throw e; - } finally { - try { - await fs.rm(tempOutputFilePath, { force: true }); - } catch (e) { - log.error( - `Failed to remove tempOutputFile ${tempOutputFilePath}`, - e, - ); - } - } -} - -function constructThumbnailGenerationCommand( - inputFilePath: string, - tempOutputFilePath: string, - maxDimension: number, - quality: number, -) { - let thumbnailGenerationCmd: string[]; - if (process.platform == "darwin") { - thumbnailGenerationCmd = SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map( - (cmdPart) => { - if (cmdPart === INPUT_PATH_PLACEHOLDER) { - return inputFilePath; - } - if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { - return tempOutputFilePath; - } - if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { - return maxDimension.toString(); - } - if (cmdPart === QUALITY_PLACEHOLDER) { - return quality.toString(); - } - return cmdPart; - }, - ); - } else if (process.platform == "linux") { - thumbnailGenerationCmd = - IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map((cmdPart) => { - if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { - return imageMagickPath(); - } - if (cmdPart === INPUT_PATH_PLACEHOLDER) { - return inputFilePath; - } - if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { - return tempOutputFilePath; - } - if (cmdPart.includes(SAMPLE_SIZE_PLACEHOLDER)) { - return cmdPart.replaceAll( - SAMPLE_SIZE_PLACEHOLDER, - (2 * maxDimension).toString(), - ); - } - if (cmdPart.includes(MAX_DIMENSION_PLACEHOLDER)) { - return cmdPart.replaceAll( - MAX_DIMENSION_PLACEHOLDER, - maxDimension.toString(), - ); - } - if (cmdPart === QUALITY_PLACEHOLDER) { - return quality.toString(); - } - return cmdPart; - }); - } else { - throw new Error(`Unsupported OS ${process.platform}`); - } - return thumbnailGenerationCmd; -} diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index c49ac67009..1505d8a96d 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -1,30 +1,32 @@ import pathToFfmpeg from "ffmpeg-static"; import fs from "node:fs/promises"; +import log from "../log"; import { withTimeout } from "../utils"; import { execAsync } from "../utils-electron"; import { deleteTempFile, makeTempFilePath } from "../utils-temp"; +/* Duplicated in the web app's code (used by the WASM FFmpeg implementation). */ const ffmpegPathPlaceholder = "FFMPEG"; const inputPathPlaceholder = "INPUT"; const outputPathPlaceholder = "OUTPUT"; /** - * Run a ffmpeg command + * Run a FFmpeg command * - * [Note: ffmpeg in Electron] + * [Note: FFmpeg in Electron] * - * There is a wasm build of ffmpeg, but that is currently 10-20 times slower + * There is a wasm build of FFmpeg, but that is currently 10-20 times slower * that the native build. That is slow enough to be unusable for our purposes. * https://ffmpegwasm.netlify.app/docs/performance * - * So the alternative is to bundle a ffmpeg binary with our app. e.g. + * So the alternative is to bundle a FFmpeg executable binary with our app. e.g. * * yarn add fluent-ffmpeg ffmpeg-static ffprobe-static * * (we only use ffmpeg-static, the rest are mentioned for completeness' sake). * - * Interestingly, Electron already bundles an ffmpeg library (it comes from the - * ffmpeg fork maintained by Chromium). + * Interestingly, Electron already bundles an binary FFmpeg library (it comes + * from the ffmpeg fork maintained by Chromium). * https://chromium.googlesource.com/chromium/third_party/ffmpeg * https://stackoverflow.com/questions/53963672/what-version-of-ffmpeg-is-bundled-inside-electron * @@ -37,8 +39,8 @@ const outputPathPlaceholder = "OUTPUT"; */ export const ffmpegExec = async ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ): Promise => { // TODO (MR): This currently copies files for both input and output. This @@ -47,18 +49,18 @@ export const ffmpegExec = async ( let inputFilePath: string; let isInputFileTemporary: boolean; - if (typeof inputDataOrPath == "string") { - inputFilePath = inputDataOrPath; - isInputFileTemporary = false; - } else { - inputFilePath = await makeTempFilePath("input" /* arbitrary */); + if (dataOrPath instanceof Uint8Array) { + inputFilePath = await makeTempFilePath(); isInputFileTemporary = true; - await fs.writeFile(inputFilePath, inputDataOrPath); + } else { + inputFilePath = dataOrPath; + isInputFileTemporary = false; } - let outputFilePath: string | undefined; + const outputFilePath = await makeTempFilePath(outputFileExtension); try { - outputFilePath = await makeTempFilePath(outputFileName); + if (dataOrPath instanceof Uint8Array) + await fs.writeFile(inputFilePath, dataOrPath); const cmd = substitutePlaceholders( command, @@ -71,8 +73,12 @@ export const ffmpegExec = async ( return fs.readFile(outputFilePath); } finally { - if (isInputFileTemporary) await deleteTempFile(inputFilePath); - if (outputFilePath) await deleteTempFile(outputFilePath); + try { + if (isInputFileTemporary) await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } } }; @@ -96,7 +102,7 @@ const substitutePlaceholders = ( /** * Return the path to the `ffmpeg` binary. * - * At runtime, the ffmpeg binary is present in a path like (macOS example): + * At runtime, the FFmpeg binary is present in a path like (macOS example): * `ente.app/Contents/Resources/app.asar.unpacked/node_modules/ffmpeg-static/ffmpeg` */ const ffmpegBinaryPath = () => { diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts new file mode 100644 index 0000000000..d8108c635f --- /dev/null +++ b/desktop/src/main/services/image.ts @@ -0,0 +1,160 @@ +/** @file Image format conversions and thumbnail generation */ + +import fs from "node:fs/promises"; +import path from "path"; +import { CustomErrorMessage } from "../../types/ipc"; +import log from "../log"; +import { execAsync, isDev } from "../utils-electron"; +import { deleteTempFile, makeTempFilePath } from "../utils-temp"; + +export const convertToJPEG = async (imageData: Uint8Array) => { + const inputFilePath = await makeTempFilePath(); + const outputFilePath = await makeTempFilePath("jpeg"); + + // Construct the command first, it may throw NotAvailable on win32. + const command = convertToJPEGCommand(inputFilePath, outputFilePath); + + try { + await fs.writeFile(inputFilePath, imageData); + await execAsync(command); + return new Uint8Array(await fs.readFile(outputFilePath)); + } finally { + try { + await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } + } +}; + +const convertToJPEGCommand = ( + inputFilePath: string, + outputFilePath: string, +) => { + switch (process.platform) { + case "darwin": + return [ + "sips", + "-s", + "format", + "jpeg", + inputFilePath, + "--out", + outputFilePath, + ]; + + case "linux": + return [ + imageMagickPath(), + inputFilePath, + "-quality", + "100%", + outputFilePath, + ]; + + default: // "win32" + throw new Error(CustomErrorMessage.NotAvailable); + } +}; + +/** Path to the Linux image-magick executable bundled with our app */ +const imageMagickPath = () => + path.join(isDev ? "build" : process.resourcesPath, "image-magick"); + +export const generateImageThumbnail = async ( + dataOrPath: Uint8Array | string, + maxDimension: number, + maxSize: number, +): Promise => { + let inputFilePath: string; + let isInputFileTemporary: boolean; + if (dataOrPath instanceof Uint8Array) { + inputFilePath = await makeTempFilePath(); + isInputFileTemporary = true; + } else { + inputFilePath = dataOrPath; + isInputFileTemporary = false; + } + + const outputFilePath = await makeTempFilePath("jpeg"); + + // Construct the command first, it may throw `NotAvailable` on win32. + let quality = 70; + let command = generateImageThumbnailCommand( + inputFilePath, + outputFilePath, + maxDimension, + quality, + ); + + try { + if (dataOrPath instanceof Uint8Array) + await fs.writeFile(inputFilePath, dataOrPath); + + let thumbnail: Uint8Array; + do { + await execAsync(command); + thumbnail = new Uint8Array(await fs.readFile(outputFilePath)); + quality -= 10; + command = generateImageThumbnailCommand( + inputFilePath, + outputFilePath, + maxDimension, + quality, + ); + } while (thumbnail.length > maxSize && quality > 50); + return thumbnail; + } finally { + try { + if (isInputFileTemporary) await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } + } +}; + +const generateImageThumbnailCommand = ( + inputFilePath: string, + outputFilePath: string, + maxDimension: number, + quality: number, +) => { + switch (process.platform) { + case "darwin": + return [ + "sips", + "-s", + "format", + "jpeg", + "-s", + "formatOptions", + `${quality}`, + "-Z", + `${maxDimension}`, + inputFilePath, + "--out", + outputFilePath, + ]; + + case "linux": + return [ + imageMagickPath(), + inputFilePath, + "-auto-orient", + "-define", + `jpeg:size=${2 * maxDimension}x${2 * maxDimension}`, + "-thumbnail", + `${maxDimension}x${maxDimension}>`, + "-unsharp", + "0x.5", + "-quality", + `${quality}`, + outputFilePath, + ]; + + default: // "win32" + throw new Error(CustomErrorMessage.NotAvailable); + } +}; diff --git a/desktop/src/main/services/ml-clip.ts b/desktop/src/main/services/ml-clip.ts index 0c466b9f66..a5f407f9e7 100644 --- a/desktop/src/main/services/ml-clip.ts +++ b/desktop/src/main/services/ml-clip.ts @@ -20,7 +20,7 @@ const cachedCLIPImageSession = makeCachedInferenceSession( ); export const clipImageEmbedding = async (jpegImageData: Uint8Array) => { - const tempFilePath = await makeTempFilePath(""); + const tempFilePath = await makeTempFilePath(); const imageStream = new Response(jpegImageData.buffer).body; await writeStream(tempFilePath, imageStream); try { diff --git a/desktop/src/main/stream.ts b/desktop/src/main/stream.ts index 8ddb80dc6a..12db786f53 100644 --- a/desktop/src/main/stream.ts +++ b/desktop/src/main/stream.ts @@ -1,15 +1,16 @@ /** * @file stream data to-from renderer using a custom protocol handler. */ -import { protocol } from "electron/main"; +import { net, protocol } from "electron/main"; import { createWriteStream, existsSync } from "node:fs"; import fs from "node:fs/promises"; import { Readable } from "node:stream"; +import { pathToFileURL } from "node:url"; import log from "./log"; /** * Register a protocol handler that we use for streaming large files between the - * main process (node) and the renderer process (browser) layer. + * main (Node.js) and renderer (Chromium) processes. * * [Note: IPC streams] * @@ -17,11 +18,14 @@ import log from "./log"; * across IPC. And passing the entire contents of the file is not feasible for * large video files because of the memory pressure the copying would entail. * - * As an alternative, we register a custom protocol handler that can provided a + * As an alternative, we register a custom protocol handler that can provides a * bi-directional stream. The renderer can stream data to the node side by * streaming the request. The node side can stream to the renderer side by * streaming the response. * + * The stream is not full duplex - while both reads and writes can be streamed, + * they need to be streamed separately. + * * See also: [Note: Transferring large amount of data over IPC] * * Depends on {@link registerPrivilegedSchemes}. @@ -29,29 +33,60 @@ import log from "./log"; export const registerStreamProtocol = () => { protocol.handle("stream", async (request: Request) => { const url = request.url; + // The request URL contains the command to run as the host, and the + // pathname of the file as the path. For example, + // + // stream://write/path/to/file + // host-pathname----- + // const { host, pathname } = new URL(url); // Convert e.g. "%20" to spaces. const path = decodeURIComponent(pathname); switch (host) { - /* stream://write/path/to/file */ - /* host-pathname----- */ + case "read": + return handleRead(path); case "write": - try { - await writeStream(path, request.body); - return new Response("", { status: 200 }); - } catch (e) { - log.error(`Failed to write stream for ${url}`, e); - return new Response( - `Failed to write stream: ${e.message}`, - { status: 500 }, - ); - } + return handleWrite(path, request); default: return new Response("", { status: 404 }); } }); }; +const handleRead = async (path: string) => { + try { + const res = await net.fetch(pathToFileURL(path).toString()); + if (res.ok) { + // net.fetch defaults to text/plain, which might be fine + // in practice, but as an extra precaution indicate that + // this is binary data. + res.headers.set("Content-Type", "application/octet-stream"); + + // Add the file's size as the Content-Length header. + const fileSize = (await fs.stat(path)).size; + res.headers.set("Content-Length", `${fileSize}`); + } + return res; + } catch (e) { + log.error(`Failed to read stream at ${path}`, e); + return new Response(`Failed to read stream: ${e.message}`, { + status: 500, + }); + } +}; + +const handleWrite = async (path: string, request: Request) => { + try { + await writeStream(path, request.body); + return new Response("", { status: 200 }); + } catch (e) { + log.error(`Failed to write stream to ${path}`, e); + return new Response(`Failed to write stream: ${e.message}`, { + status: 500, + }); + } +}; + /** * Write a (web) ReadableStream to a file at the given {@link filePath}. * diff --git a/desktop/src/main/utils-temp.ts b/desktop/src/main/utils-temp.ts index 35455e85e1..a52daf619d 100644 --- a/desktop/src/main/utils-temp.ts +++ b/desktop/src/main/utils-temp.ts @@ -13,35 +13,34 @@ const enteTempDirPath = async () => { return result; }; -const randomPrefix = (length: number) => { - const CHARACTERS = +/** Generate a random string suitable for being used as a file name prefix */ +const randomPrefix = () => { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; let result = ""; - const charactersLength = CHARACTERS.length; - for (let i = 0; i < length; i++) { - result += CHARACTERS.charAt( - Math.floor(Math.random() * charactersLength), - ); - } + for (let i = 0; i < 10; i++) + result += alphabet[Math.floor(Math.random() * alphabet.length)]; return result; }; /** - * Return the path to a temporary file with the given {@link formatSuffix}. + * Return the path to a temporary file with the given {@link suffix}. * * The function returns the path to a file in the system temp directory (in an - * Ente specific folder therin) with a random prefix and the given - * {@link formatSuffix}. It ensures that there is no existing file with the same - * name already. + * Ente specific folder therin) with a random prefix and an (optional) + * {@link extension}. + * + * It ensures that there is no existing item with the same name already. * * Use {@link deleteTempFile} to remove this file when you're done. */ -export const makeTempFilePath = async (formatSuffix: string) => { +export const makeTempFilePath = async (extension?: string) => { const tempDir = await enteTempDirPath(); + const suffix = extension ? "." + extension : ""; let result: string; do { - result = path.join(tempDir, randomPrefix(10) + "-" + formatSuffix); + result = path.join(tempDir, randomPrefix() + suffix); } while (existsSync(result)); return result; }; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index c3f964e17d..ea3cf1e054 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -124,35 +124,32 @@ const fsIsDir = (dirPath: string): Promise => // - Conversion -const convertToJPEG = ( - fileName: string, - imageData: Uint8Array, -): Promise => - ipcRenderer.invoke("convertToJPEG", fileName, imageData); +const convertToJPEG = (imageData: Uint8Array): Promise => + ipcRenderer.invoke("convertToJPEG", imageData); const generateImageThumbnail = ( - inputFile: File | ElectronFile, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ): Promise => ipcRenderer.invoke( "generateImageThumbnail", - inputFile, + dataOrPath, maxDimension, maxSize, ); const ffmpegExec = ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ): Promise => ipcRenderer.invoke( "ffmpegExec", command, - inputDataOrPath, - outputFileName, + dataOrPath, + outputFileExtension, timeoutMS, ); diff --git a/web/apps/cast/src/constants/collection.ts b/web/apps/cast/src/constants/collection.ts deleted file mode 100644 index d91cfc81d8..0000000000 --- a/web/apps/cast/src/constants/collection.ts +++ /dev/null @@ -1,24 +0,0 @@ -export enum CollectionType { - folder = "folder", - favorites = "favorites", - album = "album", - uncategorized = "uncategorized", -} - -export enum CollectionSummaryType { - folder = "folder", - favorites = "favorites", - album = "album", - archive = "archive", - trash = "trash", - uncategorized = "uncategorized", - all = "all", - outgoingShare = "outgoingShare", - incomingShareViewer = "incomingShareViewer", - incomingShareCollaborator = "incomingShareCollaborator", - sharedOnlyViaLink = "sharedOnlyViaLink", - archived = "archived", - defaultHidden = "defaultHidden", - hiddenItems = "hiddenItems", - pinned = "pinned", -} diff --git a/web/apps/cast/src/constants/file.ts b/web/apps/cast/src/constants/file.ts deleted file mode 100644 index 9be5746388..0000000000 --- a/web/apps/cast/src/constants/file.ts +++ /dev/null @@ -1,20 +0,0 @@ -export enum FILE_TYPE { - IMAGE, - VIDEO, - LIVE_PHOTO, - OTHERS, -} - -export const RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "raf", - "nef", - "psd", - "dng", - "tif", -]; diff --git a/web/apps/cast/src/constants/upload.ts b/web/apps/cast/src/constants/upload.ts index 63d044fb49..801d8a6ab4 100644 --- a/web/apps/cast/src/constants/upload.ts +++ b/web/apps/cast/src/constants/upload.ts @@ -1,6 +1,20 @@ -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { FileTypeInfo } from "types/upload"; +export const RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "raf", + "nef", + "psd", + "dng", + "tif", +]; + // list of format that were missed by type-detection for some files. export const WHITELISTED_FILE_FORMATS: FileTypeInfo[] = [ { fileType: FILE_TYPE.IMAGE, exactType: "jpeg", mimeType: "image/jpeg" }, diff --git a/web/apps/cast/src/pages/slideshow.tsx b/web/apps/cast/src/pages/slideshow.tsx index 774bbd4da9..d13a263464 100644 --- a/web/apps/cast/src/pages/slideshow.tsx +++ b/web/apps/cast/src/pages/slideshow.tsx @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import PairedSuccessfullyOverlay from "components/PairedSuccessfullyOverlay"; import { PhotoAuditorium } from "components/PhotoAuditorium"; -import { FILE_TYPE } from "constants/file"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { diff --git a/web/apps/cast/src/services/castDownloadManager.ts b/web/apps/cast/src/services/castDownloadManager.ts index 76b37c082a..40a5f25af1 100644 --- a/web/apps/cast/src/services/castDownloadManager.ts +++ b/web/apps/cast/src/services/castDownloadManager.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getCastFileURL } from "@ente/shared/network/api"; -import { FILE_TYPE } from "constants/file"; import { EnteFile } from "types/file"; import { generateStreamFromArrayBuffer } from "utils/file"; diff --git a/web/apps/cast/src/services/readerService.ts b/web/apps/cast/src/services/readerService.ts deleted file mode 100644 index 19f9bb9311..0000000000 --- a/web/apps/cast/src/services/readerService.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { convertBytesToHumanReadable } from "@/next/file"; -import log from "@/next/log"; - -export async function getUint8ArrayView(file: Blob): Promise { - try { - return new Uint8Array(await file.arrayBuffer()); - } catch (e) { - log.error( - `Failed to read file blob of size ${convertBytesToHumanReadable(file.size)}`, - e, - ); - throw e; - } -} diff --git a/web/apps/cast/src/services/typeDetectionService.ts b/web/apps/cast/src/services/typeDetectionService.ts index 5acd3844dc..883d96b0e1 100644 --- a/web/apps/cast/src/services/typeDetectionService.ts +++ b/web/apps/cast/src/services/typeDetectionService.ts @@ -1,14 +1,13 @@ -import { nameAndExtension } from "@/next/file"; +import { FILE_TYPE } from "@/media/file"; +import { convertBytesToHumanReadable, nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; -import { FILE_TYPE } from "constants/file"; import { KNOWN_NON_MEDIA_FORMATS, WHITELISTED_FILE_FORMATS, } from "constants/upload"; import FileType from "file-type"; import { FileTypeInfo } from "types/upload"; -import { getUint8ArrayView } from "./readerService"; const TYPE_VIDEO = "video"; const TYPE_IMAGE = "image"; @@ -66,6 +65,18 @@ async function extractFileType(file: File) { return getFileTypeFromBuffer(fileDataChunk); } +export async function getUint8ArrayView(file: Blob): Promise { + try { + return new Uint8Array(await file.arrayBuffer()); + } catch (e) { + log.error( + `Failed to read file blob of size ${convertBytesToHumanReadable(file.size)}`, + e, + ); + throw e; + } +} + async function getFileTypeFromBuffer(buffer: Uint8Array) { const result = await FileType.fromBuffer(buffer); if (!result?.mime) { diff --git a/web/apps/cast/src/types/collection/index.ts b/web/apps/cast/src/types/collection.ts similarity index 61% rename from web/apps/cast/src/types/collection/index.ts rename to web/apps/cast/src/types/collection.ts index f9ea9ef04b..c495937ae0 100644 --- a/web/apps/cast/src/types/collection/index.ts +++ b/web/apps/cast/src/types/collection.ts @@ -1,4 +1,3 @@ -import { CollectionSummaryType, CollectionType } from "constants/collection"; import { EnteFile } from "types/file"; import { EncryptedMagicMetadata, @@ -20,6 +19,13 @@ export interface CollectionUser { role: COLLECTION_ROLE; } +enum CollectionType { + folder = "folder", + favorites = "favorites", + album = "album", + uncategorized = "uncategorized", +} + export interface EncryptedCollection { id: number; owner: CollectionUser; @@ -32,7 +38,7 @@ export interface EncryptedCollection { type: CollectionType; attributes: collectionAttributes; sharees: CollectionUser[]; - publicURLs?: PublicURL[]; + publicURLs?: unknown; updationTime: number; isDeleted: boolean; magicMetadata: EncryptedMagicMetadata; @@ -61,54 +67,6 @@ export interface Collection // define a method on Collection interface to return the sync key as collection.id-time // this is used to store the last sync time of a collection in local storage -export interface PublicURL { - url: string; - deviceLimit: number; - validTill: number; - enableDownload: boolean; - enableCollect: boolean; - passwordEnabled: boolean; - nonce?: string; - opsLimit?: number; - memLimit?: number; -} - -export interface UpdatePublicURL { - collectionID: number; - disablePassword?: boolean; - enableDownload?: boolean; - enableCollect?: boolean; - validTill?: number; - deviceLimit?: number; - passHash?: string; - nonce?: string; - opsLimit?: number; - memLimit?: number; -} - -export interface CreatePublicAccessTokenRequest { - collectionID: number; - validTill?: number; - deviceLimit?: number; -} - -export interface EncryptedFileKey { - id: number; - encryptedKey: string; - keyDecryptionNonce: string; -} - -export interface AddToCollectionRequest { - collectionID: number; - files: EncryptedFileKey[]; -} - -export interface MoveToCollectionRequest { - fromCollectionID: number; - toCollectionID: number; - files: EncryptedFileKey[]; -} - export interface collectionAttributes { encryptedPath?: string; pathDecryptionNonce?: string; @@ -116,11 +74,6 @@ export interface collectionAttributes { export type CollectionToFileMap = Map; -export interface RemoveFromCollectionRequest { - collectionID: number; - fileIDs: number[]; -} - export interface CollectionMagicMetadataProps { visibility?: VISIBILITY_STATE; subType?: SUB_TYPE; @@ -144,16 +97,4 @@ export interface CollectionPublicMagicMetadataProps { export type CollectionPublicMagicMetadata = MagicMetadataCore; -export interface CollectionSummary { - id: number; - name: string; - type: CollectionSummaryType; - coverFile: EnteFile; - latestFile: EnteFile; - fileCount: number; - updationTime: number; - order?: number; -} - -export type CollectionSummaries = Map; export type CollectionFilesCount = Map; diff --git a/web/apps/cast/src/types/file/index.ts b/web/apps/cast/src/types/file/index.ts index 1813b5416d..f6b868c55b 100644 --- a/web/apps/cast/src/types/file/index.ts +++ b/web/apps/cast/src/types/file/index.ts @@ -64,25 +64,6 @@ export interface EnteFile isConverted?: boolean; } -export interface TrashRequest { - items: TrashRequestItems[]; -} - -export interface TrashRequestItems { - fileID: number; - collectionID: number; -} - -export interface FileWithUpdatedMagicMetadata { - file: EnteFile; - updatedMagicMetadata: FileMagicMetadata; -} - -export interface FileWithUpdatedPublicMagicMetadata { - file: EnteFile; - updatedPublicMagicMetadata: FilePublicMagicMetadata; -} - export interface FileMagicMetadataProps { visibility?: VISIBILITY_STATE; filePaths?: string[]; diff --git a/web/apps/cast/src/types/upload.ts b/web/apps/cast/src/types/upload.ts new file mode 100644 index 0000000000..91b5c33851 --- /dev/null +++ b/web/apps/cast/src/types/upload.ts @@ -0,0 +1,25 @@ +import { FILE_TYPE } from "@/media/file"; + +export interface Metadata { + title: string; + creationTime: number; + modificationTime: number; + latitude: number; + longitude: number; + fileType: FILE_TYPE; + hasStaticThumbnail?: boolean; + hash?: string; + imageHash?: string; + videoHash?: string; + localID?: number; + version?: number; + deviceFolder?: string; +} + +export interface FileTypeInfo { + fileType: FILE_TYPE; + exactType: string; + mimeType?: string; + imageType?: string; + videoType?: string; +} diff --git a/web/apps/cast/src/types/upload/index.ts b/web/apps/cast/src/types/upload/index.ts deleted file mode 100644 index 0e249846ad..0000000000 --- a/web/apps/cast/src/types/upload/index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { - B64EncryptionResult, - LocalFileAttributes, -} from "@ente/shared/crypto/types"; -import { FILE_TYPE } from "constants/file"; -import { - FilePublicMagicMetadata, - FilePublicMagicMetadataProps, - MetadataFileAttributes, - S3FileAttributes, -} from "types/file"; -import { EncryptedMagicMetadata } from "types/magicMetadata"; - -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - -export function isDataStream(object: any): object is DataStream { - return "stream" in object; -} - -export type Logger = (message: string) => void; - -export interface Metadata { - title: string; - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; - fileType: FILE_TYPE; - hasStaticThumbnail?: boolean; - hash?: string; - imageHash?: string; - videoHash?: string; - localID?: number; - version?: number; - deviceFolder?: string; -} - -export interface FileTypeInfo { - fileType: FILE_TYPE; - exactType: string; - mimeType?: string; - imageType?: string; - videoType?: string; -} - -export interface UploadURL { - url: string; - objectKey: string; -} - -export interface FileInMemory { - filedata: Uint8Array | DataStream; - thumbnail: Uint8Array; - hasStaticThumbnail: boolean; -} - -export interface FileWithMetadata - extends Omit { - metadata: Metadata; - localID: number; - pubMagicMetadata: FilePublicMagicMetadata; -} - -export interface EncryptedFile { - file: ProcessedFile; - fileKey: B64EncryptionResult; -} -export interface ProcessedFile { - file: LocalFileAttributes; - thumbnail: LocalFileAttributes; - metadata: LocalFileAttributes; - pubMagicMetadata: EncryptedMagicMetadata; - localID: number; -} -export interface BackupedFile { - file: S3FileAttributes; - thumbnail: S3FileAttributes; - metadata: MetadataFileAttributes; - pubMagicMetadata: EncryptedMagicMetadata; -} - -export interface UploadFile extends BackupedFile { - collectionID: number; - encryptedKey: string; - keyDecryptionNonce: string; -} - -export interface ParsedExtractedMetadata { - location: Location; - creationTime: number; - width: number; - height: number; -} - -export interface PublicUploadProps { - token: string; - passwordToken: string; - accessedThroughSharedURL: boolean; -} - -export interface ExtractMetadataResult { - metadata: Metadata; - publicMagicMetadata: FilePublicMagicMetadataProps; -} diff --git a/web/apps/cast/src/utils/file/index.ts b/web/apps/cast/src/utils/file.ts similarity index 88% rename from web/apps/cast/src/utils/file/index.ts rename to web/apps/cast/src/utils/file.ts index 60ec0e56e6..1f04a916df 100644 --- a/web/apps/cast/src/utils/file/index.ts +++ b/web/apps/cast/src/utils/file.ts @@ -1,7 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; -import { FILE_TYPE, RAW_FORMATS } from "constants/file"; +import { RAW_FORMATS } from "constants/upload"; import CastDownloadManager from "services/castDownloadManager"; import { getFileType } from "services/typeDetectionService"; import { @@ -103,18 +104,6 @@ export function isRawFileFromFileName(fileName: string) { return false; } -/** - * [Note: File name for local EnteFile objects] - * - * The title property in a file's metadata is the original file's name. The - * metadata of a file cannot be edited. So if later on the file's name is - * changed, then the edit is stored in the `editedName` property of the public - * metadata of the file. - * - * This function merges these edits onto the file object that we use locally. - * Effectively, post this step, the file's metadata.title can be used in lieu of - * its filename. - */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { return files.map((file) => { if (file.pubMagicMetadata?.data.editedTime) { diff --git a/web/apps/photos/src/components/EnteDateTimePicker.tsx b/web/apps/photos/src/components/EnteDateTimePicker.tsx index ee5426ebcc..e53ed65b98 100644 --- a/web/apps/photos/src/components/EnteDateTimePicker.tsx +++ b/web/apps/photos/src/components/EnteDateTimePicker.tsx @@ -5,10 +5,9 @@ import { MobileDateTimePicker, } from "@mui/x-date-pickers"; import { AdapterDateFns } from "@mui/x-date-pickers/AdapterDateFns"; -import { - MAX_EDITED_CREATION_TIME, - MIN_EDITED_CREATION_TIME, -} from "constants/file"; + +const MIN_EDITED_CREATION_TIME = new Date(1800, 0, 1); +const MAX_EDITED_CREATION_TIME = new Date(); interface Props { initialValue?: Date; diff --git a/web/apps/photos/src/components/PhotoFrame.tsx b/web/apps/photos/src/components/PhotoFrame.tsx index 90e1cf32c3..5ff70b3d45 100644 --- a/web/apps/photos/src/components/PhotoFrame.tsx +++ b/web/apps/photos/src/components/PhotoFrame.tsx @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { PHOTOS_PAGES } from "@ente/shared/constants/pages"; import { CustomError } from "@ente/shared/error"; @@ -5,7 +6,6 @@ import useMemoSingleThreaded from "@ente/shared/hooks/useMemoSingleThreaded"; import { styled } from "@mui/material"; import PhotoViewer from "components/PhotoViewer"; import { TRASH_SECTION } from "constants/collection"; -import { FILE_TYPE } from "constants/file"; import { useRouter } from "next/router"; import { GalleryContext } from "pages/gallery"; import PhotoSwipe from "photoswipe"; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx index 871da2b05f..3a5dbb6bc2 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx @@ -3,7 +3,6 @@ import { FlexWrapper } from "@ente/shared/components/Container"; import Close from "@mui/icons-material/Close"; import Done from "@mui/icons-material/Done"; import { Box, IconButton, TextField } from "@mui/material"; -import { MAX_CAPTION_SIZE } from "constants/file"; import { Formik } from "formik"; import { t } from "i18next"; import { useState } from "react"; @@ -12,6 +11,8 @@ import { changeCaption, updateExistingFilePubMetadata } from "utils/file"; import * as Yup from "yup"; import { SmallLoadingSpinner } from "../styledComponents/SmallLoadingSpinner"; +export const MAX_CAPTION_SIZE = 5000; + interface formValues { caption: string; } diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx index 1bee86c25a..c1b25e8506 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { FlexWrapper } from "@ente/shared/components/Container"; import PhotoOutlined from "@mui/icons-material/PhotoOutlined"; import VideocamOutlined from "@mui/icons-material/VideocamOutlined"; import Box from "@mui/material/Box"; -import { FILE_TYPE } from "constants/file"; import { useEffect, useState } from "react"; import { EnteFile } from "types/file"; import { makeHumanReadableStorage } from "utils/billing"; diff --git a/web/apps/photos/src/components/PhotoViewer/index.tsx b/web/apps/photos/src/components/PhotoViewer/index.tsx index 29da75e534..552c531f5d 100644 --- a/web/apps/photos/src/components/PhotoViewer/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/index.tsx @@ -16,6 +16,7 @@ import { isSupportedRawFormat, } from "utils/file"; +import { FILE_TYPE } from "@/media/file"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import AlbumOutlined from "@mui/icons-material/AlbumOutlined"; @@ -34,7 +35,6 @@ import InfoIcon from "@mui/icons-material/InfoOutlined"; import ReplayIcon from "@mui/icons-material/Replay"; import ZoomInOutlinedIcon from "@mui/icons-material/ZoomInOutlined"; import { Box, Button, styled } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; import { defaultLivePhotoDefaultOptions, photoSwipeV4Events, diff --git a/web/apps/photos/src/components/PlaceholderThumbnails.tsx b/web/apps/photos/src/components/PlaceholderThumbnails.tsx index caafbdce6f..7266bf1781 100644 --- a/web/apps/photos/src/components/PlaceholderThumbnails.tsx +++ b/web/apps/photos/src/components/PlaceholderThumbnails.tsx @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import { Overlay } from "@ente/shared/components/Container"; import PhotoOutlined from "@mui/icons-material/PhotoOutlined"; import PlayCircleOutlineOutlined from "@mui/icons-material/PlayCircleOutlineOutlined"; import { styled } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; interface Iprops { fileType: FILE_TYPE; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index d7cf151e66..87c45b59fd 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import CloseIcon from "@mui/icons-material/Close"; import { IconButton } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; import { t } from "i18next"; import memoize from "memoize-one"; import pDebounce from "p-debounce"; diff --git a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx index 6972cc1613..6dc9b851e9 100644 --- a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx +++ b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx @@ -1,4 +1,3 @@ -import log from "@/next/log"; import ChevronRight from "@mui/icons-material/ChevronRight"; import ScienceIcon from "@mui/icons-material/Science"; import { Box, DialogProps, Stack, Typography } from "@mui/material"; @@ -37,13 +36,10 @@ export default function AdvancedSettings({ open, onClose, onRootClose }) { } }; - const toggleCFProxy = async () => { - try { - appContext.setIsCFProxyDisabled(!appContext.isCFProxyDisabled); - } catch (e) { - log.error("toggleFasterUpload failed", e); - } + const toggleCFProxy = () => { + appContext.setIsCFProxyDisabled(!appContext.isCFProxyDisabled); }; + const [indexingStatus, setIndexingStatus] = useState({ indexed: 0, pending: 0, diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index 2ae077daf3..c340ee1dc3 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -1,5 +1,6 @@ import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import type { CollectionMapping, Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { isPromise } from "@ente/shared/utils"; @@ -32,11 +33,7 @@ import { SetLoading, UploadTypeSelectorIntent, } from "types/gallery"; -import { - ElectronFile, - FileWithCollection, - type FileWithCollection2, -} from "types/upload"; +import { FileWithCollection, type FileWithCollection2 } from "types/upload"; import { InProgressUpload, SegregatedFinishedUploads, @@ -492,7 +489,7 @@ export default function Uploader(props: Props) { }); throw e; } - await waitInQueueAndUploadFiles2( + await waitInQueueAndUploadFiles( filesWithCollectionToUpload, collections, ); @@ -520,24 +517,6 @@ export default function Uploader(props: Props) { await currentUploadPromise.current; }; - const waitInQueueAndUploadFiles2 = async ( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) => { - const currentPromise = currentUploadPromise.current; - currentUploadPromise.current = waitAndRun( - currentPromise, - async () => - await uploadFiles2( - filesWithCollectionToUploadIn, - collections, - uploaderName, - ), - ); - await currentUploadPromise.current; - }; - const preUploadAction = async () => { uploadManager.prepareForNewUpload(); setUploadProgressView(true); @@ -607,63 +586,6 @@ export default function Uploader(props: Props) { } }; - const uploadFiles2 = async ( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) => { - try { - log.info("uploadFiles called"); - preUploadAction(); - if ( - electron && - !isPendingDesktopUpload.current && - !watcher.isUploadRunning() - ) { - await setToUploadCollection(collections); - if (zipPaths.current) { - await electron.setPendingUploadFiles( - "zips", - zipPaths.current, - ); - zipPaths.current = null; - } - await electron.setPendingUploadFiles( - "files", - filesWithCollectionToUploadIn.map( - ({ file }) => (file as ElectronFile).path, - ), - ); - } - const shouldCloseUploadProgress = - await uploadManager.queueFilesForUpload2( - filesWithCollectionToUploadIn, - collections, - uploaderName, - ); - if (shouldCloseUploadProgress) { - closeUploadProgress(); - } - if (isElectron()) { - if (watcher.isUploadRunning()) { - await watcher.allFileUploadsDone( - filesWithCollectionToUploadIn, - collections, - ); - } else if (watcher.isSyncPaused()) { - // resume the service after user upload is done - watcher.resumePausedSync(); - } - } - } catch (e) { - log.error("failed to upload files", e); - showUserFacingError(e.message); - closeUploadProgress(); - } finally { - postUploadAction(); - } - }; - const retryFailed = async () => { try { log.info("user retrying failed upload"); diff --git a/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx b/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx index 8704258f89..b9288c59b2 100644 --- a/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx +++ b/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { Overlay } from "@ente/shared/components/Container"; import { CustomError } from "@ente/shared/error"; @@ -11,7 +12,6 @@ import { StaticThumbnail, } from "components/PlaceholderThumbnails"; import { TRASH_SECTION } from "constants/collection"; -import { FILE_TYPE } from "constants/file"; import { GAP_BTW_TILES, IMAGE_CONTAINER_MAX_WIDTH } from "constants/gallery"; import { DeduplicateContext } from "pages/deduplicate"; import { GalleryContext } from "pages/gallery"; diff --git a/web/apps/photos/src/constants/file.ts b/web/apps/photos/src/constants/file.ts deleted file mode 100644 index 46065136c9..0000000000 --- a/web/apps/photos/src/constants/file.ts +++ /dev/null @@ -1,43 +0,0 @@ -export const MIN_EDITED_CREATION_TIME = new Date(1800, 0, 1); -export const MAX_EDITED_CREATION_TIME = new Date(); - -export const MAX_EDITED_FILE_NAME_LENGTH = 100; -export const MAX_CAPTION_SIZE = 5000; - -export const TYPE_HEIC = "heic"; -export const TYPE_HEIF = "heif"; -export const TYPE_JPEG = "jpeg"; -export const TYPE_JPG = "jpg"; - -export enum FILE_TYPE { - IMAGE, - VIDEO, - LIVE_PHOTO, - OTHERS, -} - -export const RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "raf", - "nef", - "psd", - "dng", - "tif", -]; -export const SUPPORTED_RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "nef", - "psd", - "dng", - "tif", -]; diff --git a/web/apps/photos/src/constants/upload.ts b/web/apps/photos/src/constants/upload.ts index e1ee197bcf..75918fae0b 100644 --- a/web/apps/photos/src/constants/upload.ts +++ b/web/apps/photos/src/constants/upload.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants"; -import { FILE_TYPE } from "constants/file"; import { FileTypeInfo, Location } from "types/upload"; // list of format that were missed by type-detection for some files. @@ -70,11 +70,6 @@ export enum UPLOAD_STAGES { FINISH, } -export enum UPLOAD_STRATEGY { - SINGLE_COLLECTION, - COLLECTION_PER_FOLDER, -} - export enum UPLOAD_RESULT { FAILED, ALREADY_UPLOADED, diff --git a/web/apps/photos/src/services/clip-service.ts b/web/apps/photos/src/services/clip-service.ts index eae9590fd5..ab1ce928c5 100644 --- a/web/apps/photos/src/services/clip-service.ts +++ b/web/apps/photos/src/services/clip-service.ts @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; -import { FILE_TYPE } from "constants/file"; import isElectron from "is-electron"; import PQueue from "p-queue"; import { Embedding } from "types/embedding"; diff --git a/web/apps/photos/src/services/deduplicationService.ts b/web/apps/photos/src/services/deduplicationService.ts index 9d8ab399ff..f9416396f2 100644 --- a/web/apps/photos/src/services/deduplicationService.ts +++ b/web/apps/photos/src/services/deduplicationService.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import HTTPService from "@ente/shared/network/HTTPService"; import { getEndpoint } from "@ente/shared/network/api"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { FILE_TYPE } from "constants/file"; import { EnteFile } from "types/file"; import { Metadata } from "types/upload"; import { hasFileHash } from "utils/upload"; diff --git a/web/apps/photos/src/services/download/index.ts b/web/apps/photos/src/services/download/index.ts index d2ad6b1f77..7a2c359329 100644 --- a/web/apps/photos/src/services/download/index.ts +++ b/web/apps/photos/src/services/download/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { openCache, type BlobCache } from "@/next/blob-cache"; import log from "@/next/log"; @@ -8,7 +9,6 @@ import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { isPlaybackPossible } from "@ente/shared/media/video-playback"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import isElectron from "is-electron"; import * as ffmpegService from "services/ffmpeg"; import { EnteFile } from "types/file"; @@ -617,7 +617,7 @@ async function getPlayableVideo( new File([videoBlob], videoNameTitle), ); log.info(`video successfully converted ${videoNameTitle}`); - return new Blob([await mp4ConvertedVideo.arrayBuffer()]); + return new Blob([mp4ConvertedVideo]); } } catch (e) { log.error("video conversion failed", e); diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index dc7d40c70c..c949b46360 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; @@ -11,7 +12,6 @@ import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { CollectionExportNames, @@ -994,6 +994,7 @@ class ExportService { file, ); await writeStream( + electron, `${collectionExportPath}/${fileExportName}`, updatedFileStream, ); @@ -1047,6 +1048,7 @@ class ExportService { file, ); await writeStream( + electron, `${collectionExportPath}/${imageExportName}`, imageStream, ); @@ -1061,6 +1063,7 @@ class ExportService { ); try { await writeStream( + electron, `${collectionExportPath}/${videoExportName}`, videoStream, ); diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index a8c4e50689..6c13ce7fb3 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { wait } from "@ente/shared/utils"; -import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import downloadManager from "services/download"; import { getAllLocalFiles } from "services/fileService"; diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 30ab763232..b1436f17b5 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,3 +1,5 @@ +import { ElectronFile } from "@/next/types/file"; +import type { Electron } from "@/next/types/ipc"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -7,40 +9,79 @@ import { outputPathPlaceholder, } from "constants/ffmpeg"; import { NULL_LOCATION } from "constants/upload"; -import { ElectronFile, ParsedExtractedMetadata } from "types/upload"; +import { ParsedExtractedMetadata } from "types/upload"; import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; -/** Called during upload */ -export async function generateVideoThumbnail( - file: File | ElectronFile, -): Promise { - let seekTime = 1; - while (seekTime >= 0) { - try { - return await ffmpegExec( - [ - ffmpegPathPlaceholder, - "-i", - inputPathPlaceholder, - "-ss", - `00:00:0${seekTime}`, - "-vframes", - "1", - "-vf", - "scale=-1:720", - outputPathPlaceholder, - ], - file, - "thumb.jpeg", - ); - } catch (e) { - if (seekTime === 0) { - throw e; - } - } - seekTime--; +/** + * Generate a thumbnail for the given video using a wasm FFmpeg running in a web + * worker. + * + * This function is called during upload, when we need to generate thumbnails + * for the new files that the user is adding. + * + * @param blob The input video blob. + * + * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. + */ +export const generateVideoThumbnailWeb = async (blob: Blob) => + generateVideoThumbnail((seekTime: number) => + ffmpegExecWeb(genThumbnailCommand(seekTime), blob, "jpeg", 0), + ); + +const generateVideoThumbnail = async ( + thumbnailAtTime: (seekTime: number) => Promise, +) => { + try { + // Try generating thumbnail at seekTime 1 second. + return await thumbnailAtTime(1); + } catch (e) { + // If that fails, try again at the beginning. If even this throws, let + // it fail. + return await thumbnailAtTime(0); } -} +}; + +/** + * Generate a thumbnail for the given video using a native FFmpeg binary bundled + * with our desktop app. + * + * This function is called during upload, when we need to generate thumbnails + * for the new files that the user is adding. + * + * @param dataOrPath The input video's data or the path to the video on the + * user's local filesystem. See: [Note: The fileOrPath parameter to upload]. + * + * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. + */ +export const generateVideoThumbnailNative = async ( + electron: Electron, + dataOrPath: Uint8Array | string, +) => + generateVideoThumbnail((seekTime: number) => + electron.ffmpegExec( + genThumbnailCommand(seekTime), + dataOrPath, + "jpeg", + 0, + ), + ); + +const genThumbnailCommand = (seekTime: number) => [ + ffmpegPathPlaceholder, + "-i", + inputPathPlaceholder, + "-ss", + `00:00:0${seekTime}`, + "-vframes", + "1", + "-vf", + "scale=-1:720", + outputPathPlaceholder, +]; /** Called during upload */ export async function extractVideoMetadata(file: File | ElectronFile) { @@ -48,7 +89,7 @@ export async function extractVideoMetadata(file: File | ElectronFile) { // -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding // -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file - const metadata = await ffmpegExec( + const metadata = await ffmpegExec2( [ ffmpegPathPlaceholder, "-i", @@ -62,11 +103,9 @@ export async function extractVideoMetadata(file: File | ElectronFile) { outputPathPlaceholder, ], file, - `metadata.txt`, - ); - return parseFFmpegExtractedMetadata( - new Uint8Array(await metadata.arrayBuffer()), + "txt", ); + return parseFFmpegExtractedMetadata(metadata); } enum MetadataTags { @@ -135,7 +174,7 @@ function parseCreationTime(creationTime: string) { /** Called when viewing a file */ export async function convertToMP4(file: File) { - return await ffmpegExec( + return await ffmpegExec2( [ ffmpegPathPlaceholder, "-i", @@ -145,50 +184,86 @@ export async function convertToMP4(file: File) { outputPathPlaceholder, ], file, - "output.mp4", + "mp4", 30 * 1000, ); } /** - * Run the given ffmpeg command. + * Run the given FFmpeg command using a wasm FFmpeg running in a web worker. * - * If we're running in the context of our desktop app, use the ffmpeg binary we - * bundle with our desktop app to run the command. Otherwise fallback to using - * the wasm ffmpeg we link to from our web app in a web worker. - * - * As a rough ballpark, the native ffmpeg integration in the desktop app is - * 10-20x faster than the wasm one currently. See: [Note: ffmpeg in Electron]. + * As a rough ballpark, currently the native FFmpeg integration in the desktop + * app is 10-20x faster than the wasm one. See: [Note: FFmpeg in Electron]. */ -const ffmpegExec = async ( - cmd: string[], +const ffmpegExecWeb = async ( + command: string[], + blob: Blob, + outputFileExtension: string, + timeoutMs: number, +) => { + const worker = await workerFactory.lazy(); + return await worker.exec(command, blob, outputFileExtension, timeoutMs); +}; + +/** + * Run the given FFmpeg command using a native FFmpeg binary bundled with our + * desktop app. + * + * See also: {@link ffmpegExecWeb}. + */ +/* +TODO(MR): Remove me +const ffmpegExecNative = async ( + electron: Electron, + command: string[], + blob: Blob, + timeoutMs: number = 0, +) => { + const electron = globalThis.electron; + if (electron) { + const data = new Uint8Array(await blob.arrayBuffer()); + return await electron.ffmpegExec(command, data, timeoutMs); + } else { + const worker = await workerFactory.lazy(); + return await worker.exec(command, blob, timeoutMs); + } +}; +*/ + +const ffmpegExec2 = async ( + command: string[], inputFile: File | ElectronFile, - outputFilename: string, + outputFileExtension: string, timeoutMS: number = 0, -): Promise => { +) => { const electron = globalThis.electron; if (electron || false) { - /* TODO(MR): ElectronFile changes */ - // return electron.runFFmpegCmd(cmd, inputFile, outputFilename, timeoutMS); + throw new Error("WIP"); + // return electron.ffmpegExec( + // command, + // /* TODO(MR): ElectronFile changes */ + // inputFile as unknown as string, + // outputFileName, + // timeoutMS, + // ); } else { - return workerFactory - .instance() - .then((worker) => - worker.run(cmd, inputFile, outputFilename, timeoutMS), - ); + /* TODO(MR): ElectronFile changes */ + return ffmpegExecWeb( + command, + inputFile as File, + outputFileExtension, + timeoutMS, + ); } }; /** Lazily create a singleton instance of our worker */ class WorkerFactory { - private _instance: Promise>; + private instance: Promise>; - async instance() { - if (!this._instance) { - const comlinkWorker = createComlinkWorker(); - this._instance = comlinkWorker.remote; - } - return this._instance; + async lazy() { + if (!this.instance) this.instance = createComlinkWorker().remote; + return this.instance; } } diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index c5df14b224..a00c24ad3a 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; -import { FILE_TYPE } from "constants/file"; import debounce from "debounce"; import PQueue from "p-queue"; import { JobResult } from "types/common/job"; diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index a18b3c9082..d453806412 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import { MLSyncContext, MLSyncFileContext } from "types/machineLearning"; import { getLocalFileImageBitmap, diff --git a/web/apps/photos/src/services/readerService.ts b/web/apps/photos/src/services/readerService.ts index e410144cfe..e30710d5ad 100644 --- a/web/apps/photos/src/services/readerService.ts +++ b/web/apps/photos/src/services/readerService.ts @@ -1,6 +1,6 @@ import { convertBytesToHumanReadable } from "@/next/file"; import log from "@/next/log"; -import { ElectronFile } from "types/upload"; +import { ElectronFile } from "@/next/types/file"; export async function getUint8ArrayView( file: Blob | ElectronFile, diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index dfe6f20068..362e76fbef 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import * as chrono from "chrono-node"; -import { FILE_TYPE } from "constants/file"; import { t } from "i18next"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; diff --git a/web/apps/photos/src/services/typeDetectionService.ts b/web/apps/photos/src/services/typeDetectionService.ts index 5ff8f01692..598b100465 100644 --- a/web/apps/photos/src/services/typeDetectionService.ts +++ b/web/apps/photos/src/services/typeDetectionService.ts @@ -1,12 +1,13 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { CustomError } from "@ente/shared/error"; -import { FILE_TYPE } from "constants/file"; import { KNOWN_NON_MEDIA_FORMATS, WHITELISTED_FILE_FORMATS, } from "constants/upload"; import FileType, { FileTypeResult } from "file-type"; -import { ElectronFile, FileTypeInfo } from "types/upload"; +import { FileTypeInfo } from "types/upload"; import { getFileExtension } from "utils/file"; import { getUint8ArrayView } from "./readerService"; diff --git a/web/apps/photos/src/services/updateCreationTimeWithExif.ts b/web/apps/photos/src/services/updateCreationTimeWithExif.ts index 667ae44f4e..e446219150 100644 --- a/web/apps/photos/src/services/updateCreationTimeWithExif.ts +++ b/web/apps/photos/src/services/updateCreationTimeWithExif.ts @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import type { FixOption } from "components/FixCreationTime"; -import { FILE_TYPE } from "constants/file"; import { getFileType } from "services/typeDetectionService"; import { EnteFile } from "types/file"; import { diff --git a/web/apps/photos/src/services/upload/metadata.ts b/web/apps/photos/src/services/upload/metadata.ts new file mode 100644 index 0000000000..d6acb9abd3 --- /dev/null +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -0,0 +1,318 @@ +import { FILE_TYPE } from "@/media/file"; +import { getFileNameSize } from "@/next/file"; +import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; +import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; +import { CustomError } from "@ente/shared/error"; +import { + parseDateFromFusedDateString, + tryToParseDateTime, + validateAndGetCreationUnixTimeInMicroSeconds, +} from "@ente/shared/time"; +import type { DataStream } from "@ente/shared/utils/data-stream"; +import { Remote } from "comlink"; +import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload"; +import * as ffmpegService from "services/ffmpeg"; +import { getElectronFileStream, getFileStream } from "services/readerService"; +import { FilePublicMagicMetadataProps } from "types/file"; +import { + FileTypeInfo, + Metadata, + ParsedExtractedMetadata, + type LivePhotoAssets2, + type UploadAsset2, +} from "types/upload"; +import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; +import { + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + getClippedMetadataJSONMapKeyForFile, + getMetadataJSONMapKeyForFile, + type ParsedMetadataJSON, +} from "./takeout"; +import { getFileName } from "./uploadService"; + +const EXIF_TAGS_NEEDED = [ + "DateTimeOriginal", + "CreateDate", + "ModifyDate", + "GPSLatitude", + "GPSLongitude", + "GPSLatitudeRef", + "GPSLongitudeRef", + "DateCreated", + "ExifImageWidth", + "ExifImageHeight", + "ImageWidth", + "ImageHeight", + "PixelXDimension", + "PixelYDimension", + "MetadataDate", +]; + +const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { + location: NULL_LOCATION, + creationTime: null, + width: null, + height: null, +}; + +interface ExtractMetadataResult { + metadata: Metadata; + publicMagicMetadata: FilePublicMagicMetadataProps; +} + +export const extractAssetMetadata = async ( + worker: Remote, + parsedMetadataJSONMap: Map, + { isLivePhoto, file, livePhotoAssets }: UploadAsset2, + collectionID: number, + fileTypeInfo: FileTypeInfo, +): Promise => { + return isLivePhoto + ? await extractLivePhotoMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + livePhotoAssets, + ) + : await extractFileMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + file, + ); +}; + +async function extractFileMetadata( + worker: Remote, + parsedMetadataJSONMap: Map, + collectionID: number, + fileTypeInfo: FileTypeInfo, + rawFile: File | ElectronFile | string, +): Promise { + const rawFileName = getFileName(rawFile); + let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); + let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); + + if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { + key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); + googleMetadata = parsedMetadataJSONMap.get(key); + } + + const { metadata, publicMagicMetadata } = await extractMetadata( + worker, + /* TODO(MR): ElectronFile changes */ + rawFile as File | ElectronFile, + fileTypeInfo, + ); + + for (const [key, value] of Object.entries(googleMetadata ?? {})) { + if (!value) { + continue; + } + metadata[key] = value; + } + return { metadata, publicMagicMetadata }; +} + +async function extractMetadata( + worker: Remote, + receivedFile: File | ElectronFile, + fileTypeInfo: FileTypeInfo, +): Promise { + let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; + if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { + extractedMetadata = await getImageMetadata(receivedFile, fileTypeInfo); + } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { + extractedMetadata = await getVideoMetadata(receivedFile); + } + const hash = await getFileHash(worker, receivedFile); + + const metadata: Metadata = { + title: receivedFile.name, + creationTime: + extractedMetadata.creationTime ?? + extractDateFromFileName(receivedFile.name) ?? + receivedFile.lastModified * 1000, + modificationTime: receivedFile.lastModified * 1000, + latitude: extractedMetadata.location.latitude, + longitude: extractedMetadata.location.longitude, + fileType: fileTypeInfo.fileType, + hash, + }; + const publicMagicMetadata: FilePublicMagicMetadataProps = { + w: extractedMetadata.width, + h: extractedMetadata.height, + }; + return { metadata, publicMagicMetadata }; +} + +async function getImageMetadata( + receivedFile: File | ElectronFile, + fileTypeInfo: FileTypeInfo, +): Promise { + let imageMetadata = NULL_EXTRACTED_METADATA; + try { + if (!(receivedFile instanceof File)) { + receivedFile = new File( + [await receivedFile.blob()], + receivedFile.name, + { + lastModified: receivedFile.lastModified, + }, + ); + } + const exifData = await getParsedExifData( + receivedFile, + fileTypeInfo, + EXIF_TAGS_NEEDED, + ); + + imageMetadata = { + location: getEXIFLocation(exifData), + creationTime: getEXIFTime(exifData), + width: exifData?.imageWidth ?? null, + height: exifData?.imageHeight ?? null, + }; + } catch (e) { + log.error("getExifData failed", e); + } + return imageMetadata; +} + +// tries to extract date from file name if available else returns null +function extractDateFromFileName(filename: string): number { + try { + filename = filename.trim(); + let parsedDate: Date; + if (filename.startsWith("IMG-") || filename.startsWith("VID-")) { + // Whatsapp media files + // sample name IMG-20171218-WA0028.jpg + parsedDate = parseDateFromFusedDateString(filename.split("-")[1]); + } else if (filename.startsWith("Screenshot_")) { + // Screenshots on droid + // sample name Screenshot_20181227-152914.jpg + parsedDate = parseDateFromFusedDateString( + filename.replaceAll("Screenshot_", ""), + ); + } else if (filename.startsWith("signal-")) { + // signal images + // sample name :signal-2018-08-21-100217.jpg + const dateString = convertSignalNameToFusedDateString(filename); + parsedDate = parseDateFromFusedDateString(dateString); + } + if (!parsedDate) { + parsedDate = tryToParseDateTime(filename); + } + return validateAndGetCreationUnixTimeInMicroSeconds(parsedDate); + } catch (e) { + log.error("failed to extract date From FileName ", e); + return null; + } +} + +function convertSignalNameToFusedDateString(filename: string) { + const dateStringParts = filename.split("-"); + return `${dateStringParts[1]}${dateStringParts[2]}${dateStringParts[3]}-${dateStringParts[4]}`; +} + +async function getVideoMetadata(file: File | ElectronFile) { + let videoMetadata = NULL_EXTRACTED_METADATA; + try { + log.info(`getVideoMetadata called for ${getFileNameSize(file)}`); + videoMetadata = await ffmpegService.extractVideoMetadata(file); + log.info( + `videoMetadata successfully extracted ${getFileNameSize(file)}`, + ); + } catch (e) { + log.error("failed to get video metadata", e); + log.info( + `videoMetadata extracted failed ${getFileNameSize(file)} ,${ + e.message + } `, + ); + } + + return videoMetadata; +} + +async function extractLivePhotoMetadata( + worker: Remote, + parsedMetadataJSONMap: Map, + collectionID: number, + fileTypeInfo: FileTypeInfo, + livePhotoAssets: LivePhotoAssets2, +): Promise { + const imageFileTypeInfo: FileTypeInfo = { + fileType: FILE_TYPE.IMAGE, + exactType: fileTypeInfo.imageType, + }; + const { + metadata: imageMetadata, + publicMagicMetadata: imagePublicMagicMetadata, + } = await extractFileMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + imageFileTypeInfo, + livePhotoAssets.image, + ); + const videoHash = await getFileHash( + worker, + /* TODO(MR): ElectronFile changes */ + livePhotoAssets.video as File | ElectronFile, + ); + return { + metadata: { + ...imageMetadata, + title: getFileName(livePhotoAssets.image), + fileType: FILE_TYPE.LIVE_PHOTO, + imageHash: imageMetadata.hash, + videoHash: videoHash, + hash: undefined, + }, + publicMagicMetadata: imagePublicMagicMetadata, + }; +} + +async function getFileHash( + worker: Remote, + file: File | ElectronFile, +) { + try { + log.info(`getFileHash called for ${getFileNameSize(file)}`); + let filedata: DataStream; + if (file instanceof File) { + filedata = getFileStream(file, FILE_READER_CHUNK_SIZE); + } else { + filedata = await getElectronFileStream( + file, + FILE_READER_CHUNK_SIZE, + ); + } + const hashState = await worker.initChunkHashing(); + + const streamReader = filedata.stream.getReader(); + for (let i = 0; i < filedata.chunkCount; i++) { + const { done, value: chunk } = await streamReader.read(); + if (done) { + throw Error(CustomError.CHUNK_LESS_THAN_EXPECTED); + } + await worker.hashFileChunk(hashState, Uint8Array.from(chunk)); + } + const { done } = await streamReader.read(); + if (!done) { + throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); + } + const hash = await worker.completeChunkHashing(hashState); + log.info( + `file hashing completed successfully ${getFileNameSize(file)}`, + ); + return hash; + } catch (e) { + log.error("getFileHash failed", e); + log.info(`file hashing failed ${getFileNameSize(file)} ,${e.message} `); + } +} diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts deleted file mode 100644 index d1c98ff690..0000000000 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ /dev/null @@ -1,649 +0,0 @@ -import { ensureElectron } from "@/next/electron"; -import { basename, getFileNameSize } from "@/next/file"; -import log from "@/next/log"; -import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; -import { CustomError } from "@ente/shared/error"; -import { - parseDateFromFusedDateString, - tryToParseDateTime, - validateAndGetCreationUnixTimeInMicroSeconds, -} from "@ente/shared/time"; -import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; -import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload"; -import * as ffmpegService from "services/ffmpeg"; -import { getElectronFileStream, getFileStream } from "services/readerService"; -import { getFileType } from "services/typeDetectionService"; -import { FilePublicMagicMetadataProps } from "types/file"; -import { - DataStream, - ElectronFile, - ExtractMetadataResult, - FileTypeInfo, - LivePhotoAssets, - Location, - Metadata, - ParsedExtractedMetadata, - ParsedMetadataJSON, - ParsedMetadataJSONMap, - type FileWithCollection, - type FileWithCollection2, - type LivePhotoAssets2, -} from "types/upload"; -import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; -import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; -import uploadCancelService from "./uploadCancelService"; -import { extractFileMetadata, getFileName } from "./uploadService"; - -const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { - creationTime: null, - modificationTime: null, - ...NULL_LOCATION, -}; - -const EXIF_TAGS_NEEDED = [ - "DateTimeOriginal", - "CreateDate", - "ModifyDate", - "GPSLatitude", - "GPSLongitude", - "GPSLatitudeRef", - "GPSLongitudeRef", - "DateCreated", - "ExifImageWidth", - "ExifImageHeight", - "ImageWidth", - "ImageHeight", - "PixelXDimension", - "PixelYDimension", - "MetadataDate", -]; - -export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; - -export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { - location: NULL_LOCATION, - creationTime: null, - width: null, - height: null, -}; - -export async function extractMetadata( - worker: Remote, - receivedFile: File | ElectronFile, - fileTypeInfo: FileTypeInfo, -): Promise { - let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA; - if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) { - extractedMetadata = await getImageMetadata(receivedFile, fileTypeInfo); - } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { - extractedMetadata = await getVideoMetadata(receivedFile); - } - const fileHash = await getFileHash(worker, receivedFile); - - const metadata: Metadata = { - title: receivedFile.name, - creationTime: - extractedMetadata.creationTime ?? - extractDateFromFileName(receivedFile.name) ?? - receivedFile.lastModified * 1000, - modificationTime: receivedFile.lastModified * 1000, - latitude: extractedMetadata.location.latitude, - longitude: extractedMetadata.location.longitude, - fileType: fileTypeInfo.fileType, - hash: fileHash, - }; - const publicMagicMetadata: FilePublicMagicMetadataProps = { - w: extractedMetadata.width, - h: extractedMetadata.height, - }; - return { metadata, publicMagicMetadata }; -} - -export async function getImageMetadata( - receivedFile: File | ElectronFile, - fileTypeInfo: FileTypeInfo, -): Promise { - let imageMetadata = NULL_EXTRACTED_METADATA; - try { - if (!(receivedFile instanceof File)) { - receivedFile = new File( - [await receivedFile.blob()], - receivedFile.name, - { - lastModified: receivedFile.lastModified, - }, - ); - } - const exifData = await getParsedExifData( - receivedFile, - fileTypeInfo, - EXIF_TAGS_NEEDED, - ); - - imageMetadata = { - location: getEXIFLocation(exifData), - creationTime: getEXIFTime(exifData), - width: exifData?.imageWidth ?? null, - height: exifData?.imageHeight ?? null, - }; - } catch (e) { - log.error("getExifData failed", e); - } - return imageMetadata; -} - -export const getMetadataJSONMapKeyForJSON = ( - collectionID: number, - jsonFileName: string, -) => { - let title = jsonFileName.slice(0, -1 * ".json".length); - const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); - if (endsWithNumberedSuffixWithBrackets) { - title = title.slice( - 0, - -1 * endsWithNumberedSuffixWithBrackets[0].length, - ); - const [name, extension] = splitFilenameAndExtension(title); - return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; - } - return `${collectionID}-${title}`; -}; - -// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name -// so we need to use the clipped file name to get the metadataJSON file -export const getClippedMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${fileName.slice( - 0, - MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, - )}`; -}; - -export const getMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${getFileOriginalName(fileName)}`; -}; - -export async function parseMetadataJSON( - receivedFile: File | ElectronFile | string, -) { - try { - let text: string; - if (typeof receivedFile == "string") { - text = await ensureElectron().fs.readTextFile(receivedFile); - } else { - if (!(receivedFile instanceof File)) { - receivedFile = new File( - [await receivedFile.blob()], - receivedFile.name, - ); - } - text = await receivedFile.text(); - } - - return parseMetadataJSONText(text); - } catch (e) { - log.error("parseMetadataJSON failed", e); - // ignore - } -} - -export async function parseMetadataJSONText(text: string) { - const metadataJSON: object = JSON.parse(text); - - const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; - if (!metadataJSON) { - return; - } - - if ( - metadataJSON["photoTakenTime"] && - metadataJSON["photoTakenTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["photoTakenTime"]["timestamp"] * 1000000; - } else if ( - metadataJSON["creationTime"] && - metadataJSON["creationTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["creationTime"]["timestamp"] * 1000000; - } - if ( - metadataJSON["modificationTime"] && - metadataJSON["modificationTime"]["timestamp"] - ) { - parsedMetadataJSON.modificationTime = - metadataJSON["modificationTime"]["timestamp"] * 1000000; - } - let locationData: Location = NULL_LOCATION; - if ( - metadataJSON["geoData"] && - (metadataJSON["geoData"]["latitude"] !== 0.0 || - metadataJSON["geoData"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoData"]; - } else if ( - metadataJSON["geoDataExif"] && - (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || - metadataJSON["geoDataExif"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoDataExif"]; - } - if (locationData !== null) { - parsedMetadataJSON.latitude = locationData.latitude; - parsedMetadataJSON.longitude = locationData.longitude; - } - return parsedMetadataJSON; -} - -// tries to extract date from file name if available else returns null -export function extractDateFromFileName(filename: string): number { - try { - filename = filename.trim(); - let parsedDate: Date; - if (filename.startsWith("IMG-") || filename.startsWith("VID-")) { - // Whatsapp media files - // sample name IMG-20171218-WA0028.jpg - parsedDate = parseDateFromFusedDateString(filename.split("-")[1]); - } else if (filename.startsWith("Screenshot_")) { - // Screenshots on droid - // sample name Screenshot_20181227-152914.jpg - parsedDate = parseDateFromFusedDateString( - filename.replaceAll("Screenshot_", ""), - ); - } else if (filename.startsWith("signal-")) { - // signal images - // sample name :signal-2018-08-21-100217.jpg - const dateString = convertSignalNameToFusedDateString(filename); - parsedDate = parseDateFromFusedDateString(dateString); - } - if (!parsedDate) { - parsedDate = tryToParseDateTime(filename); - } - return validateAndGetCreationUnixTimeInMicroSeconds(parsedDate); - } catch (e) { - log.error("failed to extract date From FileName ", e); - return null; - } -} - -function convertSignalNameToFusedDateString(filename: string) { - const dateStringParts = filename.split("-"); - return `${dateStringParts[1]}${dateStringParts[2]}${dateStringParts[3]}-${dateStringParts[4]}`; -} - -const EDITED_FILE_SUFFIX = "-edited"; - -/* - Get the original file name for edited file to associate it to original file's metadataJSON file - as edited file doesn't have their own metadata file -*/ -function getFileOriginalName(fileName: string) { - let originalName: string = null; - const [nameWithoutExtension, extension] = - splitFilenameAndExtension(fileName); - - const isEditedFile = nameWithoutExtension.endsWith(EDITED_FILE_SUFFIX); - if (isEditedFile) { - originalName = nameWithoutExtension.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); - } else { - originalName = nameWithoutExtension; - } - if (extension) { - originalName += "." + extension; - } - return originalName; -} - -async function getVideoMetadata(file: File | ElectronFile) { - let videoMetadata = NULL_EXTRACTED_METADATA; - try { - log.info(`getVideoMetadata called for ${getFileNameSize(file)}`); - videoMetadata = await ffmpegService.extractVideoMetadata(file); - log.info( - `videoMetadata successfully extracted ${getFileNameSize(file)}`, - ); - } catch (e) { - log.error("failed to get video metadata", e); - log.info( - `videoMetadata extracted failed ${getFileNameSize(file)} ,${ - e.message - } `, - ); - } - - return videoMetadata; -} - -interface LivePhotoIdentifier { - collectionID: number; - fileType: FILE_TYPE; - name: string; - size: number; -} - -const UNDERSCORE_THREE = "_3"; -// Note: The icloud-photos-downloader library appends _HVEC to the end of the filename in case of live photos -// https://github.com/icloud-photos-downloader/icloud_photos_downloader -const UNDERSCORE_HEVC = "_HVEC"; - -export async function getLivePhotoFileType( - livePhotoAssets: LivePhotoAssets, -): Promise { - const imageFileTypeInfo = await getFileType(livePhotoAssets.image); - const videoFileTypeInfo = await getFileType(livePhotoAssets.video); - return { - fileType: FILE_TYPE.LIVE_PHOTO, - exactType: `${imageFileTypeInfo.exactType}+${videoFileTypeInfo.exactType}`, - imageType: imageFileTypeInfo.exactType, - videoType: videoFileTypeInfo.exactType, - }; -} - -export async function extractLivePhotoMetadata( - worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, - collectionID: number, - fileTypeInfo: FileTypeInfo, - livePhotoAssets: LivePhotoAssets2, -): Promise { - const imageFileTypeInfo: FileTypeInfo = { - fileType: FILE_TYPE.IMAGE, - exactType: fileTypeInfo.imageType, - }; - const { - metadata: imageMetadata, - publicMagicMetadata: imagePublicMagicMetadata, - } = await extractFileMetadata( - worker, - parsedMetadataJSONMap, - collectionID, - imageFileTypeInfo, - livePhotoAssets.image, - ); - const videoHash = await getFileHash( - worker, - /* TODO(MR): ElectronFile changes */ - livePhotoAssets.video as File | ElectronFile, - ); - return { - metadata: { - ...imageMetadata, - title: getLivePhotoName(livePhotoAssets), - fileType: FILE_TYPE.LIVE_PHOTO, - imageHash: imageMetadata.hash, - videoHash: videoHash, - hash: undefined, - }, - publicMagicMetadata: imagePublicMagicMetadata, - }; -} - -export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { - return livePhotoAssets.image.size + livePhotoAssets.video.size; -} - -export const getLivePhotoName = ({ image }: LivePhotoAssets2) => - typeof image == "string" ? basename(image) : image.name; - -export async function clusterLivePhotoFiles(mediaFiles: FileWithCollection2[]) { - try { - const analysedMediaFiles: FileWithCollection2[] = []; - mediaFiles - .sort((firstMediaFile, secondMediaFile) => - splitFilenameAndExtension( - getFileName(firstMediaFile.file), - )[0].localeCompare( - splitFilenameAndExtension( - getFileName(secondMediaFile.file), - )[0], - ), - ) - .sort( - (firstMediaFile, secondMediaFile) => - firstMediaFile.collectionID - secondMediaFile.collectionID, - ); - let index = 0; - while (index < mediaFiles.length - 1) { - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - const firstMediaFile = mediaFiles[index]; - const secondMediaFile = mediaFiles[index + 1]; - const firstFileType = - getFileTypeFromExtensionForLivePhotoClustering( - getFileName(firstMediaFile.file), - ); - const secondFileType = - getFileTypeFromExtensionForLivePhotoClustering( - getFileName(secondMediaFile.file), - ); - const firstFileIdentifier: LivePhotoIdentifier = { - collectionID: firstMediaFile.collectionID, - fileType: firstFileType, - name: getFileName(firstMediaFile.file), - /* TODO(MR): ElectronFile changes */ - size: (firstMediaFile as FileWithCollection).file.size, - }; - const secondFileIdentifier: LivePhotoIdentifier = { - collectionID: secondMediaFile.collectionID, - fileType: secondFileType, - name: getFileName(secondMediaFile.file), - /* TODO(MR): ElectronFile changes */ - size: (secondMediaFile as FileWithCollection).file.size, - }; - if ( - areFilesLivePhotoAssets( - firstFileIdentifier, - secondFileIdentifier, - ) - ) { - let imageFile: File | ElectronFile | string; - let videoFile: File | ElectronFile | string; - if ( - firstFileType === FILE_TYPE.IMAGE && - secondFileType === FILE_TYPE.VIDEO - ) { - imageFile = firstMediaFile.file; - videoFile = secondMediaFile.file; - } else { - videoFile = firstMediaFile.file; - imageFile = secondMediaFile.file; - } - const livePhotoLocalID = firstMediaFile.localID; - analysedMediaFiles.push({ - localID: livePhotoLocalID, - collectionID: firstMediaFile.collectionID, - isLivePhoto: true, - livePhotoAssets: { - image: imageFile, - video: videoFile, - }, - }); - index += 2; - } else { - analysedMediaFiles.push({ - ...firstMediaFile, - isLivePhoto: false, - }); - index += 1; - } - } - if (index === mediaFiles.length - 1) { - analysedMediaFiles.push({ - ...mediaFiles[index], - isLivePhoto: false, - }); - } - return analysedMediaFiles; - } catch (e) { - if (e.message === CustomError.UPLOAD_CANCELLED) { - throw e; - } else { - log.error("failed to cluster live photo", e); - throw e; - } - } -} - -function areFilesLivePhotoAssets( - firstFileIdentifier: LivePhotoIdentifier, - secondFileIdentifier: LivePhotoIdentifier, -) { - const haveSameCollectionID = - firstFileIdentifier.collectionID === secondFileIdentifier.collectionID; - const areNotSameFileType = - firstFileIdentifier.fileType !== secondFileIdentifier.fileType; - - let firstFileNameWithoutSuffix: string; - let secondFileNameWithoutSuffix: string; - if (firstFileIdentifier.fileType === FILE_TYPE.IMAGE) { - firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(firstFileIdentifier.name), - // Note: The Google Live Photo image file can have video extension appended as suffix, passing that to removePotentialLivePhotoSuffix to remove it - // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) - getFileExtensionWithDot(secondFileIdentifier.name), - ); - secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(secondFileIdentifier.name), - ); - } else { - firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(firstFileIdentifier.name), - ); - secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(secondFileIdentifier.name), - getFileExtensionWithDot(firstFileIdentifier.name), - ); - } - if ( - haveSameCollectionID && - isImageOrVideo(firstFileIdentifier.fileType) && - isImageOrVideo(secondFileIdentifier.fileType) && - areNotSameFileType && - firstFileNameWithoutSuffix === secondFileNameWithoutSuffix - ) { - const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB - - // checks size of live Photo assets are less than allowed limit - // I did that based on the assumption that live photo assets ideally would not be larger than LIVE_PHOTO_ASSET_SIZE_LIMIT - // also zipping library doesn't support stream as a input - if ( - firstFileIdentifier.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT && - secondFileIdentifier.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT - ) { - return true; - } else { - log.error( - `${CustomError.TOO_LARGE_LIVE_PHOTO_ASSETS} - ${JSON.stringify({ - fileSizes: [ - firstFileIdentifier.size, - secondFileIdentifier.size, - ], - })}`, - ); - } - } - return false; -} - -function removePotentialLivePhotoSuffix( - filenameWithoutExtension: string, - suffix?: string, -) { - let presentSuffix: string; - if (filenameWithoutExtension.endsWith(UNDERSCORE_THREE)) { - presentSuffix = UNDERSCORE_THREE; - } else if (filenameWithoutExtension.endsWith(UNDERSCORE_HEVC)) { - presentSuffix = UNDERSCORE_HEVC; - } else if ( - filenameWithoutExtension.endsWith(UNDERSCORE_HEVC.toLowerCase()) - ) { - presentSuffix = UNDERSCORE_HEVC.toLowerCase(); - } else if (suffix) { - if (filenameWithoutExtension.endsWith(suffix)) { - presentSuffix = suffix; - } else if (filenameWithoutExtension.endsWith(suffix.toLowerCase())) { - presentSuffix = suffix.toLowerCase(); - } - } - if (presentSuffix) { - return filenameWithoutExtension.slice(0, presentSuffix.length * -1); - } else { - return filenameWithoutExtension; - } -} - -function getFileNameWithoutExtension(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return filename; - else return filename.slice(0, lastDotPosition); -} - -function getFileExtensionWithDot(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return ""; - else return filename.slice(lastDotPosition); -} - -function splitFilenameAndExtension(filename: string): [string, string] { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return [filename, null]; - else - return [ - filename.slice(0, lastDotPosition), - filename.slice(lastDotPosition + 1), - ]; -} - -const isImageOrVideo = (fileType: FILE_TYPE) => - [FILE_TYPE.IMAGE, FILE_TYPE.VIDEO].includes(fileType); - -async function getFileHash( - worker: Remote, - file: File | ElectronFile, -) { - try { - log.info(`getFileHash called for ${getFileNameSize(file)}`); - let filedata: DataStream; - if (file instanceof File) { - filedata = getFileStream(file, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getElectronFileStream( - file, - FILE_READER_CHUNK_SIZE, - ); - } - const hashState = await worker.initChunkHashing(); - - const streamReader = filedata.stream.getReader(); - for (let i = 0; i < filedata.chunkCount; i++) { - const { done, value: chunk } = await streamReader.read(); - if (done) { - throw Error(CustomError.CHUNK_LESS_THAN_EXPECTED); - } - await worker.hashFileChunk(hashState, Uint8Array.from(chunk)); - } - const { done } = await streamReader.read(); - if (!done) { - throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); - } - const hash = await worker.completeChunkHashing(hashState); - log.info( - `file hashing completed successfully ${getFileNameSize(file)}`, - ); - return hash; - } catch (e) { - log.error("getFileHash failed", e); - log.info(`file hashing failed ${getFileNameSize(file)} ,${e.message} `); - } -} diff --git a/web/apps/photos/src/services/upload/multiPartUploadService.ts b/web/apps/photos/src/services/upload/multiPartUploadService.ts deleted file mode 100644 index 1b4442710f..0000000000 --- a/web/apps/photos/src/services/upload/multiPartUploadService.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { CustomError } from "@ente/shared/error"; -import { - FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, -} from "constants/upload"; -import { DataStream, Logger, MultipartUploadURLs } from "types/upload"; -import * as convert from "xml-js"; -import UIService from "./uiService"; -import uploadCancelService from "./uploadCancelService"; -import UploadHttpClient from "./uploadHttpClient"; -import uploadService from "./uploadService"; - -interface PartEtag { - PartNumber: number; - ETag: string; -} - -function calculatePartCount(chunkCount: number) { - const partCount = Math.ceil( - chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - ); - return partCount; -} -export async function uploadStreamUsingMultipart( - logger: Logger, - fileLocalID: number, - dataStream: DataStream, -) { - const uploadPartCount = calculatePartCount(dataStream.chunkCount); - logger(`fetching ${uploadPartCount} urls for multipart upload`); - const multipartUploadURLs = - await uploadService.fetchMultipartUploadURLs(uploadPartCount); - logger(`fetched ${uploadPartCount} urls for multipart upload`); - - const fileObjectKey = await uploadStreamInParts( - logger, - multipartUploadURLs, - dataStream.stream, - fileLocalID, - uploadPartCount, - ); - return fileObjectKey; -} - -export async function uploadStreamInParts( - logger: Logger, - multipartUploadURLs: MultipartUploadURLs, - dataStream: ReadableStream, - fileLocalID: number, - uploadPartCount: number, -) { - const streamReader = dataStream.getReader(); - const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); - const partEtags: PartEtag[] = []; - logger(`uploading file in chunks`); - for (const [ - index, - fileUploadURL, - ] of multipartUploadURLs.partURLs.entries()) { - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - const uploadChunk = await combineChunksToFormUploadPart(streamReader); - const progressTracker = UIService.trackUploadProgress( - fileLocalID, - percentPerPart, - index, - ); - let eTag = null; - if (!uploadService.getIsCFUploadProxyDisabled()) { - eTag = await UploadHttpClient.putFilePartV2( - fileUploadURL, - uploadChunk, - progressTracker, - ); - } else { - eTag = await UploadHttpClient.putFilePart( - fileUploadURL, - uploadChunk, - progressTracker, - ); - } - partEtags.push({ PartNumber: index + 1, ETag: eTag }); - } - const { done } = await streamReader.read(); - if (!done) { - throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); - } - logger(`uploading file in chunks done`); - logger(`completing multipart upload`); - await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); - logger(`completing multipart upload done`); - return multipartUploadURLs.objectKey; -} - -function getRandomProgressPerPartUpload(uploadPartCount: number) { - const percentPerPart = - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; - return percentPerPart; -} - -async function combineChunksToFormUploadPart( - streamReader: ReadableStreamDefaultReader, -) { - const combinedChunks = []; - for (let i = 0; i < FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { - const { done, value: chunk } = await streamReader.read(); - if (done) { - break; - } - for (let index = 0; index < chunk.length; index++) { - combinedChunks.push(chunk[index]); - } - } - return Uint8Array.from(combinedChunks); -} - -async function completeMultipartUpload( - partEtags: PartEtag[], - completeURL: string, -) { - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: partEtags } }, - options, - ); - if (!uploadService.getIsCFUploadProxyDisabled()) { - await UploadHttpClient.completeMultipartUploadV2(completeURL, body); - } else { - await UploadHttpClient.completeMultipartUpload(completeURL, body); - } -} diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts new file mode 100644 index 0000000000..ba6f402b43 --- /dev/null +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -0,0 +1,155 @@ +/** @file Dealing with the JSON metadata in Google Takeouts */ + +import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; +import log from "@/next/log"; +import type { ElectronFile } from "@/next/types/file"; +import { NULL_LOCATION } from "constants/upload"; +import { type Location } from "types/upload"; + +export interface ParsedMetadataJSON { + creationTime: number; + modificationTime: number; + latitude: number; + longitude: number; +} + +export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; + +export const getMetadataJSONMapKeyForJSON = ( + collectionID: number, + jsonFileName: string, +) => { + let title = jsonFileName.slice(0, -1 * ".json".length); + const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); + if (endsWithNumberedSuffixWithBrackets) { + title = title.slice( + 0, + -1 * endsWithNumberedSuffixWithBrackets[0].length, + ); + const [name, extension] = nameAndExtension(title); + return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; + } + return `${collectionID}-${title}`; +}; + +// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name +// so we need to use the clipped file name to get the metadataJSON file +export const getClippedMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${fileName.slice( + 0, + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + )}`; +}; + +export const getMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${getFileOriginalName(fileName)}`; +}; + +const EDITED_FILE_SUFFIX = "-edited"; + +/* + Get the original file name for edited file to associate it to original file's metadataJSON file + as edited file doesn't have their own metadata file +*/ +function getFileOriginalName(fileName: string) { + let originalName: string = null; + const [name, extension] = nameAndExtension(fileName); + + const isEditedFile = name.endsWith(EDITED_FILE_SUFFIX); + if (isEditedFile) { + originalName = name.slice(0, -1 * EDITED_FILE_SUFFIX.length); + } else { + originalName = name; + } + if (extension) { + originalName += "." + extension; + } + return originalName; +} + +/** Try to parse the contents of a metadata JSON file in a Google Takeout. */ +export const tryParseTakeoutMetadataJSON = async ( + receivedFile: File | ElectronFile | string, +): Promise => { + try { + let text: string; + if (typeof receivedFile == "string") { + text = await ensureElectron().fs.readTextFile(receivedFile); + } else { + if (!(receivedFile instanceof File)) { + receivedFile = new File( + [await receivedFile.blob()], + receivedFile.name, + ); + } + text = await receivedFile.text(); + } + + return parseMetadataJSONText(text); + } catch (e) { + log.error("Failed to parse takeout metadata JSON", e); + return undefined; + } +}; + +const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { + creationTime: null, + modificationTime: null, + ...NULL_LOCATION, +}; + +const parseMetadataJSONText = (text: string) => { + const metadataJSON: object = JSON.parse(text); + if (!metadataJSON) { + return undefined; + } + + const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; + + if ( + metadataJSON["photoTakenTime"] && + metadataJSON["photoTakenTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["photoTakenTime"]["timestamp"] * 1000000; + } else if ( + metadataJSON["creationTime"] && + metadataJSON["creationTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["creationTime"]["timestamp"] * 1000000; + } + if ( + metadataJSON["modificationTime"] && + metadataJSON["modificationTime"]["timestamp"] + ) { + parsedMetadataJSON.modificationTime = + metadataJSON["modificationTime"]["timestamp"] * 1000000; + } + let locationData: Location = NULL_LOCATION; + if ( + metadataJSON["geoData"] && + (metadataJSON["geoData"]["latitude"] !== 0.0 || + metadataJSON["geoData"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoData"]; + } else if ( + metadataJSON["geoDataExif"] && + (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || + metadataJSON["geoDataExif"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoDataExif"]; + } + if (locationData !== null) { + parsedMetadataJSON.latitude = locationData.latitude; + parsedMetadataJSON.longitude = locationData.longitude; + } + return parsedMetadataJSON; +}; diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 91b1ea9fb3..d8ba3a2017 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,294 +1,124 @@ -import { getFileNameSize } from "@/next/file"; +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; -import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; -import { CustomError } from "@ente/shared/error"; -import { FILE_TYPE } from "constants/file"; +import { type Electron } from "@/next/types/ipc"; +import { withTimeout } from "@ente/shared/utils"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; -import * as FFmpegService from "services/ffmpeg"; +import * as ffmpeg from "services/ffmpeg"; import { heicToJPEG } from "services/heic-convert"; -import { ElectronFile, FileTypeInfo } from "types/upload"; +import { FileTypeInfo } from "types/upload"; import { isFileHEIC } from "utils/file"; -import { getUint8ArrayView } from "../readerService"; -import { getFileName } from "./uploadService"; /** Maximum width or height of the generated thumbnail */ const maxThumbnailDimension = 720; /** Maximum size (in bytes) of the generated thumbnail */ const maxThumbnailSize = 100 * 1024; // 100 KB -const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10; -const MIN_QUALITY = 0.5; -const MAX_QUALITY = 0.7; - -const WAIT_TIME_THUMBNAIL_GENERATION = 30 * 1000; - -class ModuleState { - /** - * This will be set to true if we get an error from the Node.js side of our - * desktop app telling us that native JPEG conversion is not available for - * the current OS/arch combination. That way, we can stop pestering it again - * and again (saving an IPC round-trip). - * - * Note the double negative when it is used. - */ - isNativeThumbnailCreationNotAvailable = false; -} - -const moduleState = new ModuleState(); - -interface GeneratedThumbnail { - /** The JPEG data of the generated thumbnail */ - thumbnail: Uint8Array; - /** - * `true` if this is a fallback (all black) thumbnail we're returning since - * thumbnail generation failed for some reason. - */ - hasStaticThumbnail: boolean; -} /** - * Generate a JPEG thumbnail for the given {@link file}. + * Generate a JPEG thumbnail for the given image or video blob. * * The thumbnail has a smaller file size so that is quick to load. But more * importantly, it uses a universal file format (JPEG in our case) so that the * thumbnail itself can be opened in all clients, even those like the web client * itself that might not yet have support for more exotic formats. + * + * @param blob The image or video blob whose thumbnail we want to generate. + * + * @param fileTypeInfo The type information for the file this blob came from. + * + * @return The JPEG data of the generated thumbnail. */ -export const generateThumbnail = async ( - file: File | ElectronFile, +export const generateThumbnailWeb = async ( + blob: Blob, fileTypeInfo: FileTypeInfo, -): Promise => { - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnail(file, fileTypeInfo) - : await generateVideoThumbnail(file, fileTypeInfo); +): Promise => + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + : await generateVideoThumbnailWeb(blob); - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - log.debug(() => `Generated thumbnail for ${getFileName(file)}`); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error( - `Failed to generate thumbnail for ${getFileName(file)} with format ${fileTypeInfo.exactType}`, - e, - ); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } -}; - -/** - * A fallback, black, thumbnail for use in cases where thumbnail generation - * fails. - */ -const fallbackThumbnail = () => - Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); - -const generateImageThumbnail = async ( - file: File | ElectronFile, +const generateImageThumbnailUsingCanvas = async ( + blob: Blob, fileTypeInfo: FileTypeInfo, ) => { - let jpegData: Uint8Array | undefined; - - const electron = globalThis.electron; - const available = !moduleState.isNativeThumbnailCreationNotAvailable; - if (electron && available) { - // If we're running in our desktop app, try to make the thumbnail using - // the native tools available there-in, it'll be faster than doing it on - // the web layer. - try { - jpegData = await generateImageThumbnailInElectron(electron, file); - } catch (e) { - if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeThumbnailCreationNotAvailable = true; - } else { - log.error("Native thumbnail creation failed", e); - } - } - } - - if (!jpegData) { - jpegData = await generateImageThumbnailUsingCanvas(file, fileTypeInfo); - } - return jpegData; -}; - -const generateImageThumbnailInElectron = async ( - electron: Electron, - inputFile: File | ElectronFile, -): Promise => { - const startTime = Date.now(); - const jpegData = await electron.generateImageThumbnail( - inputFile, - maxThumbnailDimension, - maxThumbnailSize, - ); - log.debug( - () => `Native thumbnail generation took ${Date.now() - startTime} ms`, - ); - return jpegData; -}; - -async function generateImageThumbnailUsingCanvas( - file: File | ElectronFile, - fileTypeInfo: FileTypeInfo, -) { - const canvas = document.createElement("canvas"); - const canvasCTX = canvas.getContext("2d"); - - let imageURL = null; - let timeout = null; - if (isFileHEIC(fileTypeInfo.exactType)) { - log.debug(() => `Pre-converting ${getFileName(file)} to JPEG`); - const jpegBlob = await heicToJPEG(new Blob([await file.arrayBuffer()])); - file = new File([jpegBlob], file.name); + log.debug(() => `Pre-converting HEIC to JPEG for thumbnail generation`); + blob = await heicToJPEG(blob); } - let image = new Image(); - imageURL = URL.createObjectURL(new Blob([await file.arrayBuffer()])); - await new Promise((resolve, reject) => { - image.setAttribute("src", imageURL); - image.onload = () => { - try { - URL.revokeObjectURL(imageURL); - const { width, height } = scaledThumbnailDimensions( - image.width, - image.height, - maxThumbnailDimension, - ); - canvas.width = width; - canvas.height = height; - canvasCTX.drawImage(image, 0, 0, width, height); - image = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - const err = new Error(CustomError.THUMBNAIL_GENERATION_FAILED, { - cause: e, - }); - reject(err); - } - }; - timeout = setTimeout( - () => reject(new Error("Operation timed out")), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - const thumbnailBlob = await getCompressedThumbnailBlobFromCanvas(canvas); - return await getUint8ArrayView(thumbnailBlob); -} + const canvas = document.createElement("canvas"); + const canvasCtx = canvas.getContext("2d"); -async function generateVideoThumbnail( - file: File | ElectronFile, - fileTypeInfo: FileTypeInfo, -) { - let thumbnail: Uint8Array; + const imageURL = URL.createObjectURL(blob); + await withTimeout( + new Promise((resolve, reject) => { + const image = new Image(); + image.setAttribute("src", imageURL); + image.onload = () => { + try { + URL.revokeObjectURL(imageURL); + const { width, height } = scaledThumbnailDimensions( + image.width, + image.height, + maxThumbnailDimension, + ); + canvas.width = width; + canvas.height = height; + canvasCtx.drawImage(image, 0, 0, width, height); + resolve(undefined); + } catch (e) { + reject(e); + } + }; + }), + 30 * 1000, + ); + + return await compressedJPEGData(canvas); +}; + +const generateVideoThumbnailWeb = async (blob: Blob) => { try { - log.info( - `ffmpeg generateThumbnail called for ${getFileNameSize(file)}`, - ); - - const thumbnail = await FFmpegService.generateVideoThumbnail(file); - log.info( - `ffmpeg thumbnail successfully generated ${getFileNameSize(file)}`, - ); - return await getUint8ArrayView(thumbnail); + return await ffmpeg.generateVideoThumbnailWeb(blob); } catch (e) { - log.info( - `ffmpeg thumbnail generated failed ${getFileNameSize( - file, - )} error: ${e.message}`, - ); log.error( - `failed to generate thumbnail using ffmpeg for format ${fileTypeInfo.exactType}`, + `Failed to generate video thumbnail using the wasm FFmpeg web worker, will fallback to canvas`, e, ); - thumbnail = await generateVideoThumbnailUsingCanvas(file); + return generateVideoThumbnailUsingCanvas(blob); } - return thumbnail; -} +}; -async function generateVideoThumbnailUsingCanvas(file: File | ElectronFile) { +const generateVideoThumbnailUsingCanvas = async (blob: Blob) => { const canvas = document.createElement("canvas"); - const canvasCTX = canvas.getContext("2d"); + const canvasCtx = canvas.getContext("2d"); - let timeout = null; - let videoURL = null; - - let video = document.createElement("video"); - videoURL = URL.createObjectURL(new Blob([await file.arrayBuffer()])); - await new Promise((resolve, reject) => { - video.preload = "metadata"; - video.src = videoURL; - video.addEventListener("loadeddata", function () { - try { - URL.revokeObjectURL(videoURL); - if (!video) { - throw Error("video load failed"); + const videoURL = URL.createObjectURL(blob); + await withTimeout( + new Promise((resolve, reject) => { + const video = document.createElement("video"); + video.preload = "metadata"; + video.src = videoURL; + video.addEventListener("loadeddata", () => { + try { + URL.revokeObjectURL(videoURL); + const { width, height } = scaledThumbnailDimensions( + video.videoWidth, + video.videoHeight, + maxThumbnailDimension, + ); + canvas.width = width; + canvas.height = height; + canvasCtx.drawImage(video, 0, 0, width, height); + resolve(undefined); + } catch (e) { + reject(e); } - const { width, height } = scaledThumbnailDimensions( - video.videoWidth, - video.videoHeight, - maxThumbnailDimension, - ); - canvas.width = width; - canvas.height = height; - canvasCTX.drawImage(video, 0, 0, width, height); - video = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - const err = Error( - `${CustomError.THUMBNAIL_GENERATION_FAILED} err: ${e}`, - ); - log.error(CustomError.THUMBNAIL_GENERATION_FAILED, e); - reject(err); - } - }); - timeout = setTimeout( - () => reject(new Error("Operation timed out")), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - const thumbnailBlob = await getCompressedThumbnailBlobFromCanvas(canvas); - return await getUint8ArrayView(thumbnailBlob); -} - -async function getCompressedThumbnailBlobFromCanvas(canvas: HTMLCanvasElement) { - let thumbnailBlob: Blob = null; - let prevSize = Number.MAX_SAFE_INTEGER; - let quality = MAX_QUALITY; - - do { - if (thumbnailBlob) { - prevSize = thumbnailBlob.size; - } - thumbnailBlob = await new Promise((resolve) => { - canvas.toBlob( - function (blob) { - resolve(blob); - }, - "image/jpeg", - quality, - ); - }); - thumbnailBlob = thumbnailBlob ?? new Blob([]); - quality -= 0.1; - } while ( - quality >= MIN_QUALITY && - thumbnailBlob.size > maxThumbnailSize && - percentageSizeDiff(thumbnailBlob.size, prevSize) >= - MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF + }); + }), + 30 * 1000, ); - return thumbnailBlob; -} - -function percentageSizeDiff( - newThumbnailSize: number, - oldThumbnailSize: number, -) { - return ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; -} + return await compressedJPEGData(canvas); +}; /** * Compute the size of the thumbnail to create for an image with the given @@ -317,3 +147,65 @@ const scaledThumbnailDimensions = ( return { width: 0, height: 0 }; return thumbnailDimensions; }; + +const compressedJPEGData = async (canvas: HTMLCanvasElement) => { + let blob: Blob; + let prevSize = Number.MAX_SAFE_INTEGER; + let quality = 0.7; + + do { + if (blob) prevSize = blob.size; + blob = await new Promise((resolve) => { + canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality); + }); + quality -= 0.1; + } while ( + quality >= 0.5 && + blob.size > maxThumbnailSize && + percentageSizeDiff(blob.size, prevSize) >= 10 + ); + + return new Uint8Array(await blob.arrayBuffer()); +}; + +const percentageSizeDiff = ( + newThumbnailSize: number, + oldThumbnailSize: number, +) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; + +/** + * Generate a JPEG thumbnail for the given file or path using native tools. + * + * This function only works when we're running in the context of our desktop + * app, and this dependency is enforced by the need to pass the {@link electron} + * object which we use to perform IPC with the Node.js side of our desktop app. + * + * @param dataOrPath Contents of an image or video file, or the path to the + * image or video file on the user's local filesystem, whose thumbnail we want + * to generate. + * + * @param fileTypeInfo The type information for {@link dataOrPath}. + * + * @return The JPEG data of the generated thumbnail. + * + * See also {@link generateThumbnailWeb}. + */ +export const generateThumbnailNative = async ( + electron: Electron, + dataOrPath: Uint8Array | string, + fileTypeInfo: FileTypeInfo, +): Promise => + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await electron.generateImageThumbnail( + dataOrPath, + maxThumbnailDimension, + maxThumbnailSize, + ) + : ffmpeg.generateVideoThumbnailNative(electron, dataOrPath); + +/** + * A fallback, black, thumbnail for use in cases where thumbnail generation + * fails. + */ +export const fallbackThumbnail = () => + Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); diff --git a/web/apps/photos/src/services/upload/uiService.ts b/web/apps/photos/src/services/upload/uiService.ts deleted file mode 100644 index 13dd780019..0000000000 --- a/web/apps/photos/src/services/upload/uiService.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { CustomError } from "@ente/shared/error"; -import { Canceler } from "axios"; -import { - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, - UPLOAD_RESULT, - UPLOAD_STAGES, -} from "constants/upload"; -import { - FinishedUploads, - InProgressUpload, - InProgressUploads, - ProgressUpdater, - SegregatedFinishedUploads, -} from "types/upload/ui"; -import uploadCancelService from "./uploadCancelService"; - -const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec; -class UIService { - private progressUpdater: ProgressUpdater; - - // UPLOAD LEVEL STATES - private uploadStage: UPLOAD_STAGES = UPLOAD_STAGES.START; - private filenames: Map = new Map(); - private hasLivePhoto: boolean = false; - private uploadProgressView: boolean = false; - - // STAGE LEVEL STATES - private perFileProgress: number; - private filesUploadedCount: number; - private totalFilesCount: number; - private inProgressUploads: InProgressUploads = new Map(); - private finishedUploads: FinishedUploads = new Map(); - - init(progressUpdater: ProgressUpdater) { - this.progressUpdater = progressUpdater; - this.progressUpdater.setUploadStage(this.uploadStage); - this.progressUpdater.setUploadFilenames(this.filenames); - this.progressUpdater.setHasLivePhotos(this.hasLivePhoto); - this.progressUpdater.setUploadProgressView(this.uploadProgressView); - this.progressUpdater.setUploadCounter({ - finished: this.filesUploadedCount, - total: this.totalFilesCount, - }); - this.progressUpdater.setInProgressUploads( - convertInProgressUploadsToList(this.inProgressUploads), - ); - this.progressUpdater.setFinishedUploads( - segregatedFinishedUploadsToList(this.finishedUploads), - ); - } - - reset(count = 0) { - this.setTotalFileCount(count); - this.filesUploadedCount = 0; - this.inProgressUploads = new Map(); - this.finishedUploads = new Map(); - this.updateProgressBarUI(); - } - - setTotalFileCount(count: number) { - this.totalFilesCount = count; - if (count > 0) { - this.perFileProgress = 100 / this.totalFilesCount; - } else { - this.perFileProgress = 0; - } - } - - setFileProgress(key: number, progress: number) { - this.inProgressUploads.set(key, progress); - this.updateProgressBarUI(); - } - - setUploadStage(stage: UPLOAD_STAGES) { - this.uploadStage = stage; - this.progressUpdater.setUploadStage(stage); - } - - setFilenames(filenames: Map) { - this.filenames = filenames; - this.progressUpdater.setUploadFilenames(filenames); - } - - setHasLivePhoto(hasLivePhoto: boolean) { - this.hasLivePhoto = hasLivePhoto; - this.progressUpdater.setHasLivePhotos(hasLivePhoto); - } - - setUploadProgressView(uploadProgressView: boolean) { - this.uploadProgressView = uploadProgressView; - this.progressUpdater.setUploadProgressView(uploadProgressView); - } - - increaseFileUploaded() { - this.filesUploadedCount++; - this.updateProgressBarUI(); - } - - moveFileToResultList(key: number, uploadResult: UPLOAD_RESULT) { - this.finishedUploads.set(key, uploadResult); - this.inProgressUploads.delete(key); - this.updateProgressBarUI(); - } - - hasFilesInResultList() { - const finishedUploadsList = segregatedFinishedUploadsToList( - this.finishedUploads, - ); - for (const x of finishedUploadsList.values()) { - if (x.length > 0) { - return true; - } - } - return false; - } - - private updateProgressBarUI() { - const { - setPercentComplete, - setUploadCounter, - setInProgressUploads, - setFinishedUploads, - } = this.progressUpdater; - setUploadCounter({ - finished: this.filesUploadedCount, - total: this.totalFilesCount, - }); - let percentComplete = - this.perFileProgress * - (this.finishedUploads.size || this.filesUploadedCount); - if (this.inProgressUploads) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for (const [_, progress] of this.inProgressUploads) { - // filter negative indicator values during percentComplete calculation - if (progress < 0) { - continue; - } - percentComplete += (this.perFileProgress * progress) / 100; - } - } - - setPercentComplete(percentComplete); - setInProgressUploads( - convertInProgressUploadsToList(this.inProgressUploads), - ); - setFinishedUploads( - segregatedFinishedUploadsToList(this.finishedUploads), - ); - } - - trackUploadProgress( - fileLocalID: number, - percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), - index = 0, - ) { - const cancel: { exec: Canceler } = { exec: () => {} }; - const cancelTimedOutRequest = () => - cancel.exec(CustomError.REQUEST_TIMEOUT); - - const cancelCancelledUploadRequest = () => - cancel.exec(CustomError.UPLOAD_CANCELLED); - - let timeout = null; - const resetTimeout = () => { - if (timeout) { - clearTimeout(timeout); - } - timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME); - }; - return { - cancel, - onUploadProgress: (event) => { - this.inProgressUploads.set( - fileLocalID, - Math.min( - Math.round( - percentPerPart * index + - (percentPerPart * event.loaded) / event.total, - ), - 98, - ), - ); - this.updateProgressBarUI(); - if (event.loaded === event.total) { - clearTimeout(timeout); - } else { - resetTimeout(); - } - if (uploadCancelService.isUploadCancelationRequested()) { - cancelCancelledUploadRequest(); - } - }, - }; - } -} - -export default new UIService(); - -function convertInProgressUploadsToList(inProgressUploads) { - return [...inProgressUploads.entries()].map( - ([localFileID, progress]) => - ({ - localFileID, - progress, - }) as InProgressUpload, - ); -} - -function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) { - const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads; - for (const [localID, result] of finishedUploads) { - if (!segregatedFinishedUploads.has(result)) { - segregatedFinishedUploads.set(result, []); - } - segregatedFinishedUploads.get(result).push(localID); - } - return segregatedFinishedUploads; -} diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 05a336be59..3a99d35e13 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,12 +1,22 @@ +import { FILE_TYPE } from "@/media/file"; +import { potentialFileTypeFromExtension } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; +import { wait } from "@ente/shared/utils"; +import { Canceler } from "axios"; import { Remote } from "comlink"; -import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload"; +import { + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, + UPLOAD_RESULT, + UPLOAD_STAGES, +} from "constants/upload"; import isElectron from "is-electron"; import { getLocalPublicFiles, @@ -18,37 +28,241 @@ import { Collection } from "types/collection"; import { EncryptedEnteFile, EnteFile } from "types/file"; import { SetFiles } from "types/gallery"; import { - ElectronFile, FileWithCollection, - ParsedMetadataJSON, - ParsedMetadataJSONMap, PublicUploadProps, type FileWithCollection2, + type LivePhotoAssets2, } from "types/upload"; -import { ProgressUpdater } from "types/upload/ui"; -import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file"; import { - areFileWithCollectionsSame, - segregateMetadataAndMediaFiles, - segregateMetadataAndMediaFiles2, -} from "utils/upload"; + FinishedUploads, + InProgressUpload, + InProgressUploads, + ProgressUpdater, + SegregatedFinishedUploads, +} from "types/upload/ui"; +import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file"; +import { segregateMetadataAndMediaFiles } from "utils/upload"; import { getLocalFiles } from "../fileService"; import { - clusterLivePhotoFiles, getMetadataJSONMapKeyForJSON, - parseMetadataJSON, -} from "./metadataService"; -import { default as UIService, default as uiService } from "./uiService"; + tryParseTakeoutMetadataJSON, + type ParsedMetadataJSON, +} from "./takeout"; import uploadCancelService from "./uploadCancelService"; -import UploadService, { getFileName, uploader } from "./uploadService"; +import UploadService, { + assetName, + getAssetName, + getFileName, + uploader, +} from "./uploadService"; const MAX_CONCURRENT_UPLOADS = 4; +class UIService { + private progressUpdater: ProgressUpdater; + + // UPLOAD LEVEL STATES + private uploadStage: UPLOAD_STAGES = UPLOAD_STAGES.START; + private filenames: Map = new Map(); + private hasLivePhoto: boolean = false; + private uploadProgressView: boolean = false; + + // STAGE LEVEL STATES + private perFileProgress: number; + private filesUploadedCount: number; + private totalFilesCount: number; + private inProgressUploads: InProgressUploads = new Map(); + private finishedUploads: FinishedUploads = new Map(); + + init(progressUpdater: ProgressUpdater) { + this.progressUpdater = progressUpdater; + this.progressUpdater.setUploadStage(this.uploadStage); + this.progressUpdater.setUploadFilenames(this.filenames); + this.progressUpdater.setHasLivePhotos(this.hasLivePhoto); + this.progressUpdater.setUploadProgressView(this.uploadProgressView); + this.progressUpdater.setUploadCounter({ + finished: this.filesUploadedCount, + total: this.totalFilesCount, + }); + this.progressUpdater.setInProgressUploads( + convertInProgressUploadsToList(this.inProgressUploads), + ); + this.progressUpdater.setFinishedUploads( + segregatedFinishedUploadsToList(this.finishedUploads), + ); + } + + reset(count = 0) { + this.setTotalFileCount(count); + this.filesUploadedCount = 0; + this.inProgressUploads = new Map(); + this.finishedUploads = new Map(); + this.updateProgressBarUI(); + } + + setTotalFileCount(count: number) { + this.totalFilesCount = count; + if (count > 0) { + this.perFileProgress = 100 / this.totalFilesCount; + } else { + this.perFileProgress = 0; + } + } + + setFileProgress(key: number, progress: number) { + this.inProgressUploads.set(key, progress); + this.updateProgressBarUI(); + } + + setUploadStage(stage: UPLOAD_STAGES) { + this.uploadStage = stage; + this.progressUpdater.setUploadStage(stage); + } + + setFilenames(filenames: Map) { + this.filenames = filenames; + this.progressUpdater.setUploadFilenames(filenames); + } + + setHasLivePhoto(hasLivePhoto: boolean) { + this.hasLivePhoto = hasLivePhoto; + this.progressUpdater.setHasLivePhotos(hasLivePhoto); + } + + setUploadProgressView(uploadProgressView: boolean) { + this.uploadProgressView = uploadProgressView; + this.progressUpdater.setUploadProgressView(uploadProgressView); + } + + increaseFileUploaded() { + this.filesUploadedCount++; + this.updateProgressBarUI(); + } + + moveFileToResultList(key: number, uploadResult: UPLOAD_RESULT) { + this.finishedUploads.set(key, uploadResult); + this.inProgressUploads.delete(key); + this.updateProgressBarUI(); + } + + hasFilesInResultList() { + const finishedUploadsList = segregatedFinishedUploadsToList( + this.finishedUploads, + ); + for (const x of finishedUploadsList.values()) { + if (x.length > 0) { + return true; + } + } + return false; + } + + private updateProgressBarUI() { + const { + setPercentComplete, + setUploadCounter, + setInProgressUploads, + setFinishedUploads, + } = this.progressUpdater; + setUploadCounter({ + finished: this.filesUploadedCount, + total: this.totalFilesCount, + }); + let percentComplete = + this.perFileProgress * + (this.finishedUploads.size || this.filesUploadedCount); + if (this.inProgressUploads) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, progress] of this.inProgressUploads) { + // filter negative indicator values during percentComplete calculation + if (progress < 0) { + continue; + } + percentComplete += (this.perFileProgress * progress) / 100; + } + } + + setPercentComplete(percentComplete); + setInProgressUploads( + convertInProgressUploadsToList(this.inProgressUploads), + ); + setFinishedUploads( + segregatedFinishedUploadsToList(this.finishedUploads), + ); + } + + trackUploadProgress( + fileLocalID: number, + percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), + index = 0, + ) { + const cancel: { exec: Canceler } = { exec: () => {} }; + const cancelTimedOutRequest = () => + cancel.exec(CustomError.REQUEST_TIMEOUT); + + const cancelCancelledUploadRequest = () => + cancel.exec(CustomError.UPLOAD_CANCELLED); + + let timeout = null; + const resetTimeout = () => { + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(cancelTimedOutRequest, 30 * 1000 /* 30 sec */); + }; + return { + cancel, + onUploadProgress: (event) => { + this.inProgressUploads.set( + fileLocalID, + Math.min( + Math.round( + percentPerPart * index + + (percentPerPart * event.loaded) / event.total, + ), + 98, + ), + ); + this.updateProgressBarUI(); + if (event.loaded === event.total) { + clearTimeout(timeout); + } else { + resetTimeout(); + } + if (uploadCancelService.isUploadCancelationRequested()) { + cancelCancelledUploadRequest(); + } + }, + }; + } +} + +function convertInProgressUploadsToList(inProgressUploads) { + return [...inProgressUploads.entries()].map( + ([localFileID, progress]) => + ({ + localFileID, + progress, + }) as InProgressUpload, + ); +} + +function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) { + const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads; + for (const [localID, result] of finishedUploads) { + if (!segregatedFinishedUploads.has(result)) { + segregatedFinishedUploads.set(result, []); + } + segregatedFinishedUploads.get(result).push(localID); + } + return segregatedFinishedUploads; +} + class UploadManager { private cryptoWorkers = new Array< ComlinkWorker >(MAX_CONCURRENT_UPLOADS); - private parsedMetadataJSONMap: ParsedMetadataJSONMap; + private parsedMetadataJSONMap: Map; private filesToBeUploaded: FileWithCollection2[]; private remainingFiles: FileWithCollection2[] = []; private failedFiles: FileWithCollection2[]; @@ -58,20 +272,26 @@ class UploadManager { private uploadInProgress: boolean; private publicUploadProps: PublicUploadProps; private uploaderName: string; + private uiService: UIService; + private isCFUploadProxyDisabled: boolean = false; + constructor() { + this.uiService = new UIService(); + } public async init( progressUpdater: ProgressUpdater, setFiles: SetFiles, publicCollectProps: PublicUploadProps, isCFUploadProxyDisabled: boolean, ) { - UIService.init(progressUpdater); + this.uiService.init(progressUpdater); const remoteIsCFUploadProxyDisabled = await getDisableCFUploadProxyFlag(); if (remoteIsCFUploadProxyDisabled) { isCFUploadProxyDisabled = remoteIsCFUploadProxyDisabled; } - UploadService.init(publicCollectProps, isCFUploadProxyDisabled); + this.isCFUploadProxyDisabled = isCFUploadProxyDisabled; + UploadService.init(publicCollectProps); this.setFiles = setFiles; this.publicUploadProps = publicCollectProps; } @@ -91,13 +311,13 @@ class UploadManager { prepareForNewUpload() { this.resetState(); - UIService.reset(); + this.uiService.reset(); uploadCancelService.reset(); - UIService.setUploadStage(UPLOAD_STAGES.START); + this.uiService.setUploadStage(UPLOAD_STAGES.START); } showUploadProgressDialog() { - UIService.setUploadProgressView(true); + this.uiService.setUploadProgressView(true); } async updateExistingFilesAndCollections(collections: Collection[]) { @@ -128,11 +348,11 @@ class UploadManager { log.info( `received ${filesWithCollectionToUploadIn.length} files to upload`, ); - uiService.setFilenames( + this.uiService.setFilenames( new Map( filesWithCollectionToUploadIn.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + getAssetName(mediaFile), ]), ), ); @@ -141,39 +361,33 @@ class UploadManager { log.info(`has ${metadataJSONFiles.length} metadata json files`); log.info(`has ${mediaFiles.length} media files`); if (metadataJSONFiles.length) { - UIService.setUploadStage( + this.uiService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.parseMetadataJSONFiles(metadataJSONFiles); - - UploadService.setParsedMetadataJSONMap( - this.parsedMetadataJSONMap, - ); } + if (mediaFiles.length) { - log.info(`clusterLivePhotoFiles started`); - const analysedMediaFiles = - await clusterLivePhotoFiles(mediaFiles); - log.info(`clusterLivePhotoFiles ended`); - log.info( - `got live photos: ${ - mediaFiles.length !== analysedMediaFiles.length - }`, - ); - uiService.setFilenames( + const clusteredMediaFiles = clusterLivePhotos(mediaFiles); + + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } + + this.uiService.setFilenames( new Map( - analysedMediaFiles.map((mediaFile) => [ + clusteredMediaFiles.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + assetName(mediaFile), ]), ), ); - UIService.setHasLivePhoto( - mediaFiles.length !== analysedMediaFiles.length, + this.uiService.setHasLivePhoto( + mediaFiles.length !== clusteredMediaFiles.length, ); - await this.uploadMediaFiles(analysedMediaFiles); + await this.uploadMediaFiles(clusteredMediaFiles); } } catch (e) { if (e.message === CustomError.UPLOAD_CANCELLED) { @@ -186,105 +400,14 @@ class UploadManager { throw e; } } finally { - UIService.setUploadStage(UPLOAD_STAGES.FINISH); + this.uiService.setUploadStage(UPLOAD_STAGES.FINISH); for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { this.cryptoWorkers[i]?.terminate(); } this.uploadInProgress = false; } try { - if (!UIService.hasFilesInResultList()) { - return true; - } else { - return false; - } - } catch (e) { - log.error(" failed to return shouldCloseProgressBar", e); - return false; - } - } - - public async queueFilesForUpload2( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) { - try { - if (this.uploadInProgress) { - throw Error("can't run multiple uploads at once"); - } - this.uploadInProgress = true; - await this.updateExistingFilesAndCollections(collections); - this.uploaderName = uploaderName; - log.info( - `received ${filesWithCollectionToUploadIn.length} files to upload`, - ); - uiService.setFilenames( - new Map( - filesWithCollectionToUploadIn.map((mediaFile) => [ - mediaFile.localID, - UploadService.getAssetName(mediaFile), - ]), - ), - ); - const { metadataJSONFiles, mediaFiles } = - segregateMetadataAndMediaFiles2(filesWithCollectionToUploadIn); - log.info(`has ${metadataJSONFiles.length} metadata json files`); - log.info(`has ${mediaFiles.length} media files`); - if (metadataJSONFiles.length) { - UIService.setUploadStage( - UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, - ); - await this.parseMetadataJSONFiles(metadataJSONFiles); - - UploadService.setParsedMetadataJSONMap( - this.parsedMetadataJSONMap, - ); - } - if (mediaFiles.length) { - log.info(`clusterLivePhotoFiles started`); - const analysedMediaFiles = - await clusterLivePhotoFiles(mediaFiles); - log.info(`clusterLivePhotoFiles ended`); - log.info( - `got live photos: ${ - mediaFiles.length !== analysedMediaFiles.length - }`, - ); - uiService.setFilenames( - new Map( - analysedMediaFiles.map((mediaFile) => [ - mediaFile.localID, - UploadService.getAssetName(mediaFile), - ]), - ), - ); - - UIService.setHasLivePhoto( - mediaFiles.length !== analysedMediaFiles.length, - ); - - await this.uploadMediaFiles(analysedMediaFiles); - } - } catch (e) { - if (e.message === CustomError.UPLOAD_CANCELLED) { - if (isElectron()) { - this.remainingFiles = []; - await cancelRemainingUploads(); - } - } else { - log.error("uploading failed with error", e); - throw e; - } - } finally { - UIService.setUploadStage(UPLOAD_STAGES.FINISH); - for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { - this.cryptoWorkers[i]?.terminate(); - } - this.uploadInProgress = false; - } - try { - if (!UIService.hasFilesInResultList()) { + if (!this.uiService.hasFilesInResultList()) { return true; } else { return false; @@ -299,7 +422,7 @@ class UploadManager { try { log.info(`parseMetadataJSONFiles function executed `); - UIService.reset(metadataFiles.length); + this.uiService.reset(metadataFiles.length); for (const { file, collectionID } of metadataFiles) { const name = getFileName(file); @@ -310,13 +433,14 @@ class UploadManager { log.info(`parsing metadata json file ${name}`); - const parsedMetadataJSON = await parseMetadataJSON(file); - if (parsedMetadataJSON) { + const metadataJSON = + await tryParseTakeoutMetadataJSON(file); + if (metadataJSON) { this.parsedMetadataJSONMap.set( getMetadataJSONMapKeyForJSON(collectionID, name), - parsedMetadataJSON && { ...parsedMetadataJSON }, + metadataJSON && { ...metadataJSON }, ); - UIService.increaseFileUploaded(); + this.uiService.increaseFileUploaded(); } log.info(`successfully parsed metadata json file ${name}`); } catch (e) { @@ -347,11 +471,11 @@ class UploadManager { this.remainingFiles = [...this.remainingFiles, ...mediaFiles]; } - UIService.reset(mediaFiles.length); + this.uiService.reset(mediaFiles.length); await UploadService.setFileCount(mediaFiles.length); - UIService.setUploadStage(UPLOAD_STAGES.UPLOADING); + this.uiService.setUploadStage(UPLOAD_STAGES.UPLOADING); const uploadProcesses = []; for ( @@ -367,6 +491,8 @@ class UploadManager { } private async uploadNextFileInQueue(worker: Remote) { + const uiService = this.uiService; + while (this.filesToBeUploaded.length > 0) { if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); @@ -375,11 +501,27 @@ class UploadManager { const { collectionID } = fileWithCollection; const collection = this.collections.get(collectionID); fileWithCollection = { ...fileWithCollection, collection }; + + uiService.setFileProgress(fileWithCollection.localID, 0); + await wait(0); + const { fileUploadResult, uploadedFile } = await uploader( worker, this.existingFiles, fileWithCollection, + this.parsedMetadataJSONMap, this.uploaderName, + this.isCFUploadProxyDisabled, + ( + fileLocalID: number, + percentPerPart?: number, + index?: number, + ) => + uiService.trackUploadProgress( + fileLocalID, + percentPerPart, + index, + ), ); const finalUploadResult = await this.postUploadTask( @@ -388,11 +530,11 @@ class UploadManager { fileWithCollection, ); - UIService.moveFileToResultList( + this.uiService.moveFileToResultList( fileWithCollection.localID, finalUploadResult, ); - UIService.increaseFileUploaded(); + this.uiService.increaseFileUploaded(); UploadService.reducePendingUploadCount(); } } @@ -407,7 +549,7 @@ class UploadManager { log.info( `post upload action -> fileUploadResult: ${fileUploadResult} uploadedFile present ${!!uploadedFile}`, ); - await this.updateElectronRemainingFiles(fileWithCollection); + await this.removeFromPendingUploads(fileWithCollection); switch (fileUploadResult) { case UPLOAD_RESULT.FAILED: case UPLOAD_RESULT.BLOCKED: @@ -483,7 +625,7 @@ class UploadManager { public cancelRunningUpload() { log.info("user cancelled running upload"); - UIService.setUploadStage(UPLOAD_STAGES.CANCELLING); + this.uiService.setUploadStage(UPLOAD_STAGES.CANCELLING); uploadCancelService.requestUploadCancelation(); } @@ -510,12 +652,10 @@ class UploadManager { this.setFiles((files) => sortFiles([...files, decryptedFile])); } - private async updateElectronRemainingFiles( - fileWithCollection: FileWithCollection2, - ) { + private async removeFromPendingUploads(file: FileWithCollection2) { if (isElectron()) { this.remainingFiles = this.remainingFiles.filter( - (file) => !areFileWithCollectionsSame(file, fileWithCollection), + (f) => f.localID != file.localID, ); await updatePendingUploads(this.remainingFiles); } @@ -570,3 +710,167 @@ const cancelRemainingUploads = async () => { await electron.setPendingUploadFiles("zips", []); await electron.setPendingUploadFiles("files", []); }; + +/** + * The data needed by {@link clusterLivePhotos} to do its thing. + * + * As files progress through stages, they get more and more bits tacked on to + * them. These types document the journey. + */ +type ClusterableFile = { + localID: number; + collectionID: number; + // fileOrPath: File | ElectronFile | string; + file: File | ElectronFile | string; +}; + +type ClusteredFile = ClusterableFile & { + isLivePhoto: boolean; + livePhotoAssets?: LivePhotoAssets2; +}; + +/** + * Go through the given files, combining any sibling image + video assets into a + * single live photo when appropriate. + */ +const clusterLivePhotos = (mediaFiles: ClusterableFile[]) => { + const result: ClusteredFile[] = []; + mediaFiles + .sort((f, g) => + nameAndExtension(getFileName(f.file))[0].localeCompare( + nameAndExtension(getFileName(g.file))[0], + ), + ) + .sort((f, g) => f.collectionID - g.collectionID); + let index = 0; + while (index < mediaFiles.length - 1) { + const f = mediaFiles[index]; + const g = mediaFiles[index + 1]; + const fFileName = getFileName(f.file); + const gFileName = getFileName(g.file); + const fFileType = potentialFileTypeFromExtension(fFileName); + const gFileType = potentialFileTypeFromExtension(gFileName); + const fa: PotentialLivePhotoAsset = { + fileName: fFileName, + fileType: fFileType, + collectionID: f.collectionID, + /* TODO(MR): ElectronFile changes */ + size: (f as FileWithCollection).file.size, + }; + const ga: PotentialLivePhotoAsset = { + fileName: gFileName, + fileType: gFileType, + collectionID: g.collectionID, + /* TODO(MR): ElectronFile changes */ + size: (g as FileWithCollection).file.size, + }; + if (areLivePhotoAssets(fa, ga)) { + result.push({ + localID: f.localID, + collectionID: f.collectionID, + isLivePhoto: true, + livePhotoAssets: { + image: fFileType == FILE_TYPE.IMAGE ? f.file : g.file, + video: fFileType == FILE_TYPE.IMAGE ? g.file : f.file, + }, + }); + index += 2; + } else { + result.push({ + ...f, + isLivePhoto: false, + }); + index += 1; + } + } + if (index === mediaFiles.length - 1) { + result.push({ + ...mediaFiles[index], + isLivePhoto: false, + }); + } + return result; +}; + +interface PotentialLivePhotoAsset { + fileName: string; + fileType: FILE_TYPE; + collectionID: number; + size: number; +} + +const areLivePhotoAssets = ( + f: PotentialLivePhotoAsset, + g: PotentialLivePhotoAsset, +) => { + if (f.collectionID != g.collectionID) return false; + + const [fName, fExt] = nameAndExtension(f.fileName); + const [gName, gExt] = nameAndExtension(g.fileName); + + let fPrunedName: string; + let gPrunedName: string; + if (f.fileType == FILE_TYPE.IMAGE && g.fileType == FILE_TYPE.VIDEO) { + fPrunedName = removePotentialLivePhotoSuffix( + fName, + // A Google Live Photo image file can have video extension appended + // as suffix, so we pass that to removePotentialLivePhotoSuffix to + // remove it. + // + // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) + gExt ? `.${gExt}` : undefined, + ); + gPrunedName = removePotentialLivePhotoSuffix(gName); + } else if (f.fileType == FILE_TYPE.VIDEO && g.fileType == FILE_TYPE.IMAGE) { + fPrunedName = removePotentialLivePhotoSuffix(fName); + gPrunedName = removePotentialLivePhotoSuffix( + gName, + fExt ? `.${fExt}` : undefined, + ); + } else { + return false; + } + + if (fPrunedName != gPrunedName) return false; + + // Also check that the size of an individual Live Photo asset is less than + // an (arbitrary) limit. This should be true in practice as the videos for a + // live photo are a few seconds long. Further on, the zipping library that + // we use doesn't support stream as a input. + + const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ + if (f.size > maxAssetSize || g.size > maxAssetSize) { + log.info( + `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, + ); + return false; + } + + return true; +}; + +const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { + const suffix_3 = "_3"; + + // The icloud-photos-downloader library appends _HVEC to the end of the + // filename in case of live photos. + // + // https://github.com/icloud-photos-downloader/icloud_photos_downloader + const suffix_hvec = "_HVEC"; + + let foundSuffix: string | undefined; + if (name.endsWith(suffix_3)) { + foundSuffix = suffix_3; + } else if ( + name.endsWith(suffix_hvec) || + name.endsWith(suffix_hvec.toLowerCase()) + ) { + foundSuffix = suffix_hvec; + } else if (suffix) { + if (name.endsWith(suffix) || name.endsWith(suffix.toLowerCase())) { + foundSuffix = suffix; + } + } + + return foundSuffix ? name.slice(0, foundSuffix.length * -1) : name; +}; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 78953bd241..e208fd4447 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,104 +1,74 @@ +import { FILE_TYPE } from "@/media/file"; import { encodeLivePhoto } from "@/media/live-photo"; -import { - basename, - convertBytesToHumanReadable, - getFileNameSize, -} from "@/next/file"; +import { ensureElectron } from "@/next/electron"; +import { basename } from "@/next/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; +import { CustomErrorMessage } from "@/next/types/ipc"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; -import { - B64EncryptionResult, - EncryptionResult, -} from "@ente/shared/crypto/types"; +import { EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; -import { wait } from "@ente/shared/utils"; +import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import { + FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE, + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UPLOAD_RESULT, } from "constants/upload"; import { addToCollection } from "services/collectionService"; -import { Collection } from "types/collection"; import { EnteFile, - FilePublicMagicMetadata, - FilePublicMagicMetadataProps, + type FilePublicMagicMetadata, + type FilePublicMagicMetadataProps, } from "types/file"; import { EncryptedMagicMetadata } from "types/magicMetadata"; import { BackupedFile, - DataStream, - ElectronFile, EncryptedFile, - ExtractMetadataResult, FileInMemory, FileTypeInfo, FileWithMetadata, - Logger, - ParsedMetadataJSON, - ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, UploadAsset, UploadFile, UploadURL, - isDataStream, type FileWithCollection2, type LivePhotoAssets, + type LivePhotoAssets2, + type Metadata, type UploadAsset2, } from "types/upload"; import { getNonEmptyMagicMetadataProps, updateMagicMetadata, } from "utils/magicMetadata"; -import { findMatchingExistingFiles } from "utils/upload"; -import { - getElectronFileStream, - getFileStream, - getUint8ArrayView, -} from "../readerService"; +import { readStream } from "utils/native-stream"; +import { hasFileHash } from "utils/upload"; +import * as convert from "xml-js"; +import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; -import { - MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, - extractLivePhotoMetadata, - extractMetadata, - getClippedMetadataJSONMapKeyForFile, - getLivePhotoFileType, - getLivePhotoName, - getLivePhotoSize, - getMetadataJSONMapKeyForFile, -} from "./metadataService"; -import { uploadStreamUsingMultipart } from "./multiPartUploadService"; +import { extractAssetMetadata } from "./metadata"; import publicUploadHttpClient from "./publicUploadHttpClient"; -import { generateThumbnail } from "./thumbnail"; -import UIService from "./uiService"; +import type { ParsedMetadataJSON } from "./takeout"; +import { + fallbackThumbnail, + generateThumbnailNative, + generateThumbnailWeb, +} from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; /** Upload files to cloud storage */ class UploadService { private uploadURLs: UploadURL[] = []; - private parsedMetadataJSONMap: ParsedMetadataJSONMap = new Map< - string, - ParsedMetadataJSON - >(); - - private uploaderName: string; - private pendingUploadCount: number = 0; - private publicUploadProps: PublicUploadProps = undefined; - private isCFUploadProxyDisabled: boolean = false; - - init( - publicUploadProps: PublicUploadProps, - isCFUploadProxyDisabled: boolean, - ) { + init(publicUploadProps: PublicUploadProps) { this.publicUploadProps = publicUploadProps; - this.isCFUploadProxyDisabled = isCFUploadProxyDisabled; } async setFileCount(fileCount: number) { @@ -106,190 +76,18 @@ class UploadService { await this.preFetchUploadURLs(); } - setParsedMetadataJSONMap(parsedMetadataJSONMap: ParsedMetadataJSONMap) { - this.parsedMetadataJSONMap = parsedMetadataJSONMap; - } - - setUploaderName(uploaderName: string) { - this.uploaderName = uploaderName; - } - - getUploaderName() { - return this.uploaderName; - } - - getIsCFUploadProxyDisabled() { - return this.isCFUploadProxyDisabled; - } - reducePendingUploadCount() { this.pendingUploadCount--; } - getAssetSize({ isLivePhoto, file, livePhotoAssets }: UploadAsset) { - return isLivePhoto - ? getLivePhotoSize(livePhotoAssets) - : getFileSize(file); - } - - getAssetName({ isLivePhoto, file, livePhotoAssets }: UploadAsset2) { - return isLivePhoto - ? getLivePhotoName(livePhotoAssets) - : getFileName(file); - } - - getAssetFileType({ isLivePhoto, file, livePhotoAssets }: UploadAsset) { - return isLivePhoto - ? getLivePhotoFileType(livePhotoAssets) - : getFileType(file); - } - - async readAsset( - fileTypeInfo: FileTypeInfo, - { isLivePhoto, file, livePhotoAssets }: UploadAsset, - ) { - return isLivePhoto - ? await readLivePhoto(fileTypeInfo, livePhotoAssets) - : await readFile(fileTypeInfo, file); - } - - async extractAssetMetadata( - worker: Remote, - { isLivePhoto, file, livePhotoAssets }: UploadAsset2, - collectionID: number, - fileTypeInfo: FileTypeInfo, - ): Promise { - return isLivePhoto - ? extractLivePhotoMetadata( - worker, - this.parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - livePhotoAssets, - ) - : await extractFileMetadata( - worker, - this.parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - file, - ); - } - - constructPublicMagicMetadata( - publicMagicMetadataProps: FilePublicMagicMetadataProps, - ) { - return constructPublicMagicMetadata(publicMagicMetadataProps); - } - - async encryptAsset( - worker: Remote, - file: FileWithMetadata, - encryptionKey: string, - ): Promise { - return encryptFile(worker, file, encryptionKey); - } - - async uploadToBucket( - logger: Logger, - file: ProcessedFile, - ): Promise { - try { - let fileObjectKey: string = null; - logger("uploading file to bucket"); - if (isDataStream(file.file.encryptedData)) { - logger("uploading using multipart"); - fileObjectKey = await uploadStreamUsingMultipart( - logger, - file.localID, - file.file.encryptedData, - ); - logger("uploading using multipart done"); - } else { - logger("uploading using single part"); - const progressTracker = UIService.trackUploadProgress( - file.localID, - ); - const fileUploadURL = await this.getUploadURL(); - if (!this.isCFUploadProxyDisabled) { - logger("uploading using cf proxy"); - fileObjectKey = await UploadHttpClient.putFileV2( - fileUploadURL, - file.file.encryptedData as Uint8Array, - progressTracker, - ); - } else { - logger("uploading directly to s3"); - fileObjectKey = await UploadHttpClient.putFile( - fileUploadURL, - file.file.encryptedData as Uint8Array, - progressTracker, - ); - } - logger("uploading using single part done"); - } - logger("uploading thumbnail to bucket"); - const thumbnailUploadURL = await this.getUploadURL(); - let thumbnailObjectKey: string = null; - if (!this.isCFUploadProxyDisabled) { - thumbnailObjectKey = await UploadHttpClient.putFileV2( - thumbnailUploadURL, - file.thumbnail.encryptedData, - null, - ); - } else { - thumbnailObjectKey = await UploadHttpClient.putFile( - thumbnailUploadURL, - file.thumbnail.encryptedData, - null, - ); - } - logger("uploading thumbnail to bucket done"); - - const backupedFile: BackupedFile = { - file: { - decryptionHeader: file.file.decryptionHeader, - objectKey: fileObjectKey, - }, - thumbnail: { - decryptionHeader: file.thumbnail.decryptionHeader, - objectKey: thumbnailObjectKey, - }, - metadata: file.metadata, - pubMagicMetadata: file.pubMagicMetadata, - }; - return backupedFile; - } catch (e) { - if (e.message !== CustomError.UPLOAD_CANCELLED) { - log.error("error uploading to bucket", e); - } - throw e; - } - } - - getUploadFile( - collection: Collection, - backupedFile: BackupedFile, - fileKey: B64EncryptionResult, - ): UploadFile { - const uploadFile: UploadFile = { - collectionID: collection.id, - encryptedKey: fileKey.encryptedData, - keyDecryptionNonce: fileKey.nonce, - ...backupedFile, - }; - uploadFile; - return uploadFile; - } - - private async getUploadURL() { + async getUploadURL() { if (this.uploadURLs.length === 0 && this.pendingUploadCount) { await this.fetchUploadURLs(); } return this.uploadURLs.pop(); } - public async preFetchUploadURLs() { + private async preFetchUploadURLs() { try { await this.fetchUploadURLs(); // checking for any subscription related errors @@ -345,9 +143,491 @@ const uploadService = new UploadService(); export default uploadService; -export async function constructPublicMagicMetadata( +/** + * A function that can be called to obtain a "progressTracker" that then is + * directly fed to axios to both cancel the upload if needed, and update the + * progress status. + * + * Enhancement: The return value needs to be typed. + */ +type MakeProgressTracker = ( + fileLocalID: number, + percentPerPart?: number, + index?: number, +) => unknown; + +interface UploadResponse { + fileUploadResult: UPLOAD_RESULT; + uploadedFile?: EnteFile; +} + +export const uploader = async ( + worker: Remote, + existingFiles: EnteFile[], + fileWithCollection: FileWithCollection2, + parsedMetadataJSONMap: Map, + uploaderName: string, + isCFUploadProxyDisabled: boolean, + makeProgessTracker: MakeProgressTracker, +): Promise => { + const name = assetName(fileWithCollection); + log.info(`Uploading ${name}`); + + const abortIfCancelled = () => { + if (uploadCancelService.isUploadCancelationRequested()) + throw Error(CustomError.UPLOAD_CANCELLED); + }; + + const { collection, localID, ...uploadAsset2 } = fileWithCollection; + /* TODO(MR): ElectronFile changes */ + const uploadAsset = uploadAsset2 as UploadAsset; + let fileTypeInfo: FileTypeInfo; + let fileSize: number; + try { + const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB + + fileSize = getAssetSize(uploadAsset); + if (fileSize >= maxFileSize) { + return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; + } + fileTypeInfo = await getAssetFileType(uploadAsset); + + const { metadata, publicMagicMetadata } = await extractAssetMetadata( + worker, + parsedMetadataJSONMap, + uploadAsset, + collection.id, + fileTypeInfo, + ); + + const matches = existingFiles.filter((file) => + areFilesSame(file.metadata, metadata), + ); + + const anyMatch = matches?.length > 0 ? matches[0] : undefined; + + if (anyMatch) { + const matchInSameCollection = matches.find( + (f) => f.collectionID == collection.id, + ); + if (matchInSameCollection) { + return { + fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED, + uploadedFile: matchInSameCollection, + }; + } else { + // Any of the matching files can be used to add a symlink. + const symlink = Object.assign({}, anyMatch); + symlink.collectionID = collection.id; + await addToCollection(collection, [symlink]); + return { + fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK, + uploadedFile: symlink, + }; + } + } + + abortIfCancelled(); + + const file = await readAsset(fileTypeInfo, uploadAsset); + + if (file.hasStaticThumbnail) metadata.hasStaticThumbnail = true; + + const pubMagicMetadata = await constructPublicMagicMetadata({ + ...publicMagicMetadata, + uploaderName, + }); + + abortIfCancelled(); + + const fileWithMetadata: FileWithMetadata = { + localID, + filedata: file.filedata, + thumbnail: file.thumbnail, + metadata, + pubMagicMetadata, + }; + + const encryptedFile = await encryptFile( + worker, + fileWithMetadata, + collection.key, + ); + + abortIfCancelled(); + + const backupedFile = await uploadToBucket( + encryptedFile.file, + makeProgessTracker, + isCFUploadProxyDisabled, + abortIfCancelled, + ); + + const uploadedFile = await uploadService.uploadFile({ + collectionID: collection.id, + encryptedKey: encryptedFile.fileKey.encryptedData, + keyDecryptionNonce: encryptedFile.fileKey.nonce, + ...backupedFile, + }); + + return { + fileUploadResult: metadata.hasStaticThumbnail + ? UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL + : UPLOAD_RESULT.UPLOADED, + uploadedFile: uploadedFile, + }; + } catch (e) { + if (e.message == CustomError.UPLOAD_CANCELLED) { + log.info(`Upload for ${name} cancelled`); + } else if (e.message == CustomError.UNSUPPORTED_FILE_FORMAT) { + log.info(`Not uploading ${name}: unsupported file format`); + } else { + log.error(`Upload failed for ${name}`, e); + } + + const error = handleUploadError(e); + switch (error.message) { + case CustomError.ETAG_MISSING: + return { fileUploadResult: UPLOAD_RESULT.BLOCKED }; + case CustomError.UNSUPPORTED_FILE_FORMAT: + return { fileUploadResult: UPLOAD_RESULT.UNSUPPORTED }; + case CustomError.FILE_TOO_LARGE: + return { + fileUploadResult: + UPLOAD_RESULT.LARGER_THAN_AVAILABLE_STORAGE, + }; + default: + return { fileUploadResult: UPLOAD_RESULT.FAILED }; + } + } +}; + +export const getFileName = (file: File | ElectronFile | string) => + typeof file == "string" ? basename(file) : file.name; + +function getFileSize(file: File | ElectronFile) { + return file.size; +} +export const getAssetName = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset) => + isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file); + +export const assetName = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset2) => + isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file); + +const getAssetSize = ({ isLivePhoto, file, livePhotoAssets }: UploadAsset) => { + return isLivePhoto ? getLivePhotoSize(livePhotoAssets) : getFileSize(file); +}; + +const getLivePhotoSize = (livePhotoAssets: LivePhotoAssets) => { + return livePhotoAssets.image.size + livePhotoAssets.video.size; +}; + +const getAssetFileType = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset) => { + return isLivePhoto + ? getLivePhotoFileType(livePhotoAssets) + : getFileType(file); +}; + +const getLivePhotoFileType = async ( + livePhotoAssets: LivePhotoAssets, +): Promise => { + const imageFileTypeInfo = await getFileType(livePhotoAssets.image); + const videoFileTypeInfo = await getFileType(livePhotoAssets.video); + return { + fileType: FILE_TYPE.LIVE_PHOTO, + exactType: `${imageFileTypeInfo.exactType}+${videoFileTypeInfo.exactType}`, + imageType: imageFileTypeInfo.exactType, + videoType: videoFileTypeInfo.exactType, + }; +}; + +const readAsset = async ( + fileTypeInfo: FileTypeInfo, + { isLivePhoto, file, livePhotoAssets }: UploadAsset2, +) => { + return isLivePhoto + ? await readLivePhoto(livePhotoAssets, fileTypeInfo) + : await readImageOrVideo(file, fileTypeInfo); +}; + +// TODO(MR): Merge with the uploader +class ModuleState { + /** + * This will be set to true if we get an error from the Node.js side of our + * desktop app telling us that native image thumbnail generation is not + * available for the current OS/arch combination. + * + * That way, we can stop pestering it again and again (saving an IPC + * round-trip). + * + * Note the double negative when it is used. + */ + isNativeImageThumbnailGenerationNotAvailable = false; +} + +const moduleState = new ModuleState(); + +/** + * Read the given file or path into an in-memory representation. + * + * [Note: The fileOrPath parameter to upload] + * + * The file can be either a web + * [File](https://developer.mozilla.org/en-US/docs/Web/API/File) or the absolute + * path to a file on desk. When and why, read on. + * + * This code gets invoked in two contexts: + * + * 1. web: the normal mode, when we're running in as a web app in the browser. + * + * 2. desktop: when we're running inside our desktop app. + * + * In the web context, we'll always get a File, since within the browser we + * cannot programmatically construct paths to or arbitrarily access files on the + * user's filesystem. Note that even if we were to have an absolute path at + * hand, we cannot programmatically create such File objects to arbitrary + * absolute paths on user's local filesystem for security reasons. + * + * So in the web context, this will always be a File we get as a result of an + * explicit user interaction (e.g. drag and drop). + * + * In the desktop context, this can be either a File or a path. + * + * 1. If the user provided us this file via some user interaction (say a drag + * and a drop), this'll still be a File. + * + * 2. However, when running in the desktop app we have the ability to access + * absolute paths on the user's file system. For example, if the user asks us + * to watch certain folders on their disk for changes, we'll be able to pick + * up new images being added, and in such cases, the parameter here will be a + * path. Another example is when resuming an previously interrupted upload - + * we'll only have the path at hand in such cases, not the File object. + * + * The advantage of the File object is that the browser has already read it into + * memory for us. The disadvantage comes in the case where we need to + * communicate with the native Node.js layer of our desktop app. Since this + * communication happens over IPC, the File's contents need to be serialized and + * copied, which is a bummer for large videos etc. + * + * So when we do have a path, we first try to see if we can perform IPC using + * the path itself (e.g. when generating thumbnails). Eventually, we'll need to + * read the file once when we need to encrypt and upload it, but if we're smart + * we can do all the rest of the IPC operations using the path itself, and for + * the read during upload using a streaming IPC mechanism. + */ +const readFileOrPath = async ( + fileOrPath: File | string, +): Promise<{ dataOrStream: Uint8Array | DataStream; fileSize: number }> => { + let dataOrStream: Uint8Array | DataStream; + let fileSize: number; + + if (fileOrPath instanceof File) { + const file = fileOrPath; + fileSize = file.size; + dataOrStream = + fileSize > MULTIPART_PART_SIZE + ? getFileStream(file, FILE_READER_CHUNK_SIZE) + : new Uint8Array(await file.arrayBuffer()); + } else { + const path = fileOrPath; + const { response, size } = await readStream(ensureElectron(), path); + fileSize = size; + if (size > MULTIPART_PART_SIZE) { + const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); + dataOrStream = { stream: response.body, chunkCount }; + } else { + dataOrStream = new Uint8Array(await response.arrayBuffer()); + } + } + + return { dataOrStream, fileSize }; +}; + +/** + * Augment the given {@link dataOrStream} with thumbnail information. + * + * This is a companion method for {@link readFileOrPath}, and can be used to + * convert the result of {@link readFileOrPath} into an {@link FileInMemory}. + * + * Note: The returned dataOrStream might be different from the one that we + * provide to it. + */ +const withThumbnail = async ( + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, + dataOrStream: Uint8Array | DataStream, + fileSize: number, +): Promise => { + let thumbnail: Uint8Array | undefined; + let hasStaticThumbnail = false; + + const electron = globalThis.electron; + const notAvailable = + fileTypeInfo.fileType == FILE_TYPE.IMAGE && + moduleState.isNativeImageThumbnailGenerationNotAvailable; + + // 1. Native thumbnail generation. + if (electron && !notAvailable) { + try { + if (fileOrPath instanceof File) { + if (dataOrStream instanceof Uint8Array) { + thumbnail = await generateThumbnailNative( + electron, + dataOrStream, + fileTypeInfo, + ); + } else { + // This was large enough to need streaming, and trying to + // read it into memory or copying over IPC might cause us to + // run out of memory. So skip the native generation for it, + // instead let it get processed by the browser based + // thumbnailer (case 2). + } + } else { + thumbnail = await generateThumbnailNative( + electron, + fileOrPath, + fileTypeInfo, + ); + } + } catch (e) { + if (e.message == CustomErrorMessage.NotAvailable) { + moduleState.isNativeImageThumbnailGenerationNotAvailable = true; + } else { + log.error("Native thumbnail generation failed", e); + } + } + } + + if (!thumbnail) { + let blob: Blob | undefined; + if (fileOrPath instanceof File) { + // 2. Browser based thumbnail generation for `File`s. + blob = fileOrPath; + } else { + // 3. Browser based thumbnail generation for paths. + if (dataOrStream instanceof Uint8Array) { + blob = new Blob([dataOrStream]); + } else { + // Read the stream into memory. Don't try this fallback for huge + // files though lest we run out of memory. + if (fileSize < 100 * 1024 * 1024 /* 100 MB */) { + const data = await readEntireStream(dataOrStream.stream); + // The Readable stream cannot be read twice, so also + // overwrite the stream with the data we read. + dataOrStream = data; + blob = new Blob([data]); + } else { + // There isn't a normal scenario where this should happen. + // Case 1, should've already worked, and the only known + // reason it'd have been skipped is for image files on + // Windows, but those should be less than 100 MB. + // + // So don't risk running out of memory for a case we don't + // comprehend. + log.error( + `Not using browser based thumbnail generation fallback for large file at path ${fileOrPath}`, + ); + } + } + } + + try { + thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); + } catch (e) { + log.error("Web thumbnail creation failed", e); + } + } + + if (!thumbnail) { + thumbnail = fallbackThumbnail(); + hasStaticThumbnail = true; + } + + return { + filedata: dataOrStream, + thumbnail, + hasStaticThumbnail, + }; +}; + +/** + * Read the entirety of a readable stream. + * + * It is not recommended to use this for large (say, multi-hundred MB) files. It + * is provided as a syntactic shortcut for cases where we already know that the + * size of the stream will be reasonable enough to be read in its entirety + * without us running out of memory. + */ +const readEntireStream = async (stream: ReadableStream) => + new Uint8Array(await new Response(stream).arrayBuffer()); + +const readImageOrVideo = async ( + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, +) => { + const { dataOrStream, fileSize } = await readFileOrPath(fileOrPath); + return withThumbnail(fileOrPath, fileTypeInfo, dataOrStream, fileSize); +}; + +const readLivePhoto = async ( + livePhotoAssets: LivePhotoAssets2, + fileTypeInfo: FileTypeInfo, +) => { + const readImage = await readFileOrPath(livePhotoAssets.image); + const { + filedata: imageDataOrStream, + thumbnail, + hasStaticThumbnail, + } = await withThumbnail( + livePhotoAssets.image, + { + exactType: fileTypeInfo.imageType, + fileType: FILE_TYPE.IMAGE, + }, + readImage.dataOrStream, + readImage.fileSize, + ); + const readVideo = await readFileOrPath(livePhotoAssets.video); + + // We can revisit this later, but the existing code always read the + // full files into memory here, and to avoid changing the rest of + // the scaffolding retain the same behaviour. + // + // This is a reasonable assumption too, since the videos + // corresponding to live photos are only a couple of seconds long. + const toData = async (dataOrStream: Uint8Array | DataStream) => + dataOrStream instanceof Uint8Array + ? dataOrStream + : await readEntireStream(dataOrStream.stream); + + return { + filedata: await encodeLivePhoto({ + imageFileName: getFileName(livePhotoAssets.image), + imageData: await toData(imageDataOrStream), + videoFileName: getFileName(livePhotoAssets.video), + videoData: await toData(readVideo.dataOrStream), + }), + thumbnail, + hasStaticThumbnail, + }; +}; + +const constructPublicMagicMetadata = async ( publicMagicMetadataProps: FilePublicMagicMetadataProps, -): Promise { +): Promise => { const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps( publicMagicMetadataProps, ); @@ -356,108 +636,7 @@ export async function constructPublicMagicMetadata( return null; } return await updateMagicMetadata(publicMagicMetadataProps); -} - -function getFileSize(file: File | ElectronFile) { - return file.size; -} - -export const getFileName = (file: File | ElectronFile | string) => - typeof file == "string" ? basename(file) : file.name; - -async function readFile( - fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile, -): Promise { - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - rawFile, - fileTypeInfo, - ); - log.info(`reading file data ${getFileNameSize(rawFile)} `); - let filedata: Uint8Array | DataStream; - if (!(rawFile instanceof File)) { - if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = await getElectronFileStream( - rawFile, - FILE_READER_CHUNK_SIZE, - ); - } else { - filedata = await getUint8ArrayView(rawFile); - } - } else if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getUint8ArrayView(rawFile); - } - - log.info(`read file data successfully ${getFileNameSize(rawFile)} `); - - return { - filedata, - thumbnail, - hasStaticThumbnail, - }; -} - -async function readLivePhoto( - fileTypeInfo: FileTypeInfo, - livePhotoAssets: LivePhotoAssets, -) { - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - livePhotoAssets.image, - { - exactType: fileTypeInfo.imageType, - fileType: FILE_TYPE.IMAGE, - }, - ); - - const imageData = await getUint8ArrayView(livePhotoAssets.image); - - const videoData = await getUint8ArrayView(livePhotoAssets.video); - - return { - filedata: await encodeLivePhoto({ - imageFileName: livePhotoAssets.image.name, - imageData, - videoFileName: livePhotoAssets.video.name, - videoData, - }), - thumbnail, - hasStaticThumbnail, - }; -} - -export async function extractFileMetadata( - worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, - collectionID: number, - fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile | string, -): Promise { - const rawFileName = getFileName(rawFile); - let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); - let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); - - if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { - key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); - googleMetadata = parsedMetadataJSONMap.get(key); - } - - const { metadata, publicMagicMetadata } = await extractMetadata( - worker, - /* TODO(MR): ElectronFile changes */ - rawFile as File | ElectronFile, - fileTypeInfo, - ); - - for (const [key, value] of Object.entries(googleMetadata ?? {})) { - if (!value) { - continue; - } - metadata[key] = value; - } - return { metadata, publicMagicMetadata }; -} +}; async function encryptFile( worker: Remote, @@ -555,186 +734,206 @@ async function encryptFileStream( }; } -interface UploadResponse { - fileUploadResult: UPLOAD_RESULT; - uploadedFile?: EnteFile; -} +/** + * Return true if the two files, as represented by their metadata, are same. + * + * Note that the metadata includes the hash of the file's contents (when + * available), so this also in effect compares the contents of the files, not + * just the "meta" information about them. + */ +const areFilesSame = (f: Metadata, g: Metadata) => + hasFileHash(f) && hasFileHash(g) + ? areFilesSameHash(f, g) + : areFilesSameNoHash(f, g); -export async function uploader( - worker: Remote, - existingFiles: EnteFile[], - fileWithCollection: FileWithCollection2, - uploaderName: string, -): Promise { - const { collection, localID, ...uploadAsset2 } = fileWithCollection; - /* TODO(MR): ElectronFile changes */ - const uploadAsset = uploadAsset2 as UploadAsset; - const fileNameSize = `${uploadService.getAssetName( - fileWithCollection, - )}_${convertBytesToHumanReadable(uploadService.getAssetSize(uploadAsset))}`; +const areFilesSameHash = (f: Metadata, g: Metadata) => { + if (f.fileType !== g.fileType || f.title !== g.title) { + return false; + } + if (f.fileType === FILE_TYPE.LIVE_PHOTO) { + return f.imageHash === g.imageHash && f.videoHash === g.videoHash; + } else { + return f.hash === g.hash; + } +}; - log.info(`uploader called for ${fileNameSize}`); - UIService.setFileProgress(localID, 0); - await wait(0); - let fileTypeInfo: FileTypeInfo; - let fileSize: number; +/** + * Older files that were uploaded before we introduced hashing will not have + * hashes, so retain and use the logic we used back then for such files. + */ +const areFilesSameNoHash = (f: Metadata, g: Metadata) => { + /* + * The maximum difference in the creation/modification times of two similar + * files is set to 1 second. This is because while uploading files in the + * web - browsers and users could have set reduced precision of file times + * to prevent timing attacks and fingerprinting. + * + * See: + * https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision + */ + const oneSecond = 1e6; + return ( + f.fileType === g.fileType && + Math.abs(f.creationTime - g.creationTime) < oneSecond && + Math.abs(f.modificationTime - g.modificationTime) < oneSecond && + f.title === g.title + ); +}; + +const uploadToBucket = async ( + file: ProcessedFile, + makeProgessTracker: MakeProgressTracker, + isCFUploadProxyDisabled: boolean, + abortIfCancelled: () => void, +): Promise => { try { - const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB + let fileObjectKey: string = null; - fileSize = uploadService.getAssetSize(uploadAsset); - if (fileSize >= maxFileSize) { - return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; - } - log.info(`getting filetype for ${fileNameSize}`); - fileTypeInfo = await uploadService.getAssetFileType(uploadAsset); - log.info( - `got filetype for ${fileNameSize} - ${JSON.stringify(fileTypeInfo)}`, - ); - - log.info(`extracting metadata ${fileNameSize}`); - const { metadata, publicMagicMetadata } = - await uploadService.extractAssetMetadata( - worker, - uploadAsset, - collection.id, - fileTypeInfo, + if (isDataStream(file.file.encryptedData)) { + fileObjectKey = await uploadStreamUsingMultipart( + file.localID, + file.file.encryptedData, + makeProgessTracker, + isCFUploadProxyDisabled, + abortIfCancelled, ); - - const matchingExistingFiles = findMatchingExistingFiles( - existingFiles, - metadata, - ); - log.debug( - () => - `matchedFileList: ${matchingExistingFiles - .map((f) => `${f.id}-${f.metadata.title}`) - .join(",")}`, - ); - if (matchingExistingFiles?.length) { - const matchingExistingFilesCollectionIDs = - matchingExistingFiles.map((e) => e.collectionID); - log.debug( - () => - `matched file collectionIDs:${matchingExistingFilesCollectionIDs} - and collectionID:${collection.id}`, - ); - if (matchingExistingFilesCollectionIDs.includes(collection.id)) { - log.info( - `file already present in the collection , skipped upload for ${fileNameSize}`, + } else { + const progressTracker = makeProgessTracker(file.localID); + const fileUploadURL = await uploadService.getUploadURL(); + if (!isCFUploadProxyDisabled) { + fileObjectKey = await UploadHttpClient.putFileV2( + fileUploadURL, + file.file.encryptedData as Uint8Array, + progressTracker, ); - const sameCollectionMatchingExistingFile = - matchingExistingFiles.find( - (f) => f.collectionID === collection.id, - ); - return { - fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED, - uploadedFile: sameCollectionMatchingExistingFile, - }; } else { - log.info( - `same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize} ,adding symlink`, + fileObjectKey = await UploadHttpClient.putFile( + fileUploadURL, + file.file.encryptedData as Uint8Array, + progressTracker, ); - // any of the matching file can used to add a symlink - const resultFile = Object.assign({}, matchingExistingFiles[0]); - resultFile.collectionID = collection.id; - await addToCollection(collection, [resultFile]); - return { - fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK, - uploadedFile: resultFile, - }; } } - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`reading asset ${fileNameSize}`); - - const file = await uploadService.readAsset(fileTypeInfo, uploadAsset); - - if (file.hasStaticThumbnail) { - metadata.hasStaticThumbnail = true; - } - - const pubMagicMetadata = - await uploadService.constructPublicMagicMetadata({ - ...publicMagicMetadata, - uploaderName, - }); - - const fileWithMetadata: FileWithMetadata = { - localID, - filedata: file.filedata, - thumbnail: file.thumbnail, - metadata, - pubMagicMetadata, - }; - - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`encryptAsset ${fileNameSize}`); - const encryptedFile = await uploadService.encryptAsset( - worker, - fileWithMetadata, - collection.key, - ); - - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`uploadToBucket ${fileNameSize}`); - const logger: Logger = (message: string) => { - log.info(message, `fileNameSize: ${fileNameSize}`); - }; - const backupedFile: BackupedFile = await uploadService.uploadToBucket( - logger, - encryptedFile.file, - ); - - const uploadFile: UploadFile = uploadService.getUploadFile( - collection, - backupedFile, - encryptedFile.fileKey, - ); - log.info(`uploading file to server ${fileNameSize}`); - - const uploadedFile = await uploadService.uploadFile(uploadFile); - - log.info(`${fileNameSize} successfully uploaded`); - - return { - fileUploadResult: metadata.hasStaticThumbnail - ? UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL - : UPLOAD_RESULT.UPLOADED, - uploadedFile: uploadedFile, - }; - } catch (e) { - log.info(`upload failed for ${fileNameSize} ,error: ${e.message}`); - if ( - e.message !== CustomError.UPLOAD_CANCELLED && - e.message !== CustomError.UNSUPPORTED_FILE_FORMAT - ) { - log.error( - `file upload failed - ${JSON.stringify({ - fileFormat: fileTypeInfo?.exactType, - fileSize: convertBytesToHumanReadable(fileSize), - })}`, - e, + const thumbnailUploadURL = await uploadService.getUploadURL(); + let thumbnailObjectKey: string = null; + if (!isCFUploadProxyDisabled) { + thumbnailObjectKey = await UploadHttpClient.putFileV2( + thumbnailUploadURL, + file.thumbnail.encryptedData, + null, + ); + } else { + thumbnailObjectKey = await UploadHttpClient.putFile( + thumbnailUploadURL, + file.thumbnail.encryptedData, + null, ); } - const error = handleUploadError(e); - switch (error.message) { - case CustomError.ETAG_MISSING: - return { fileUploadResult: UPLOAD_RESULT.BLOCKED }; - case CustomError.UNSUPPORTED_FILE_FORMAT: - return { fileUploadResult: UPLOAD_RESULT.UNSUPPORTED }; - case CustomError.FILE_TOO_LARGE: - return { - fileUploadResult: - UPLOAD_RESULT.LARGER_THAN_AVAILABLE_STORAGE, - }; - default: - return { fileUploadResult: UPLOAD_RESULT.FAILED }; + + const backupedFile: BackupedFile = { + file: { + decryptionHeader: file.file.decryptionHeader, + objectKey: fileObjectKey, + }, + thumbnail: { + decryptionHeader: file.thumbnail.decryptionHeader, + objectKey: thumbnailObjectKey, + }, + metadata: file.metadata, + pubMagicMetadata: file.pubMagicMetadata, + }; + return backupedFile; + } catch (e) { + if (e.message !== CustomError.UPLOAD_CANCELLED) { + log.error("error uploading to bucket", e); + } + throw e; + } +}; + +interface PartEtag { + PartNumber: number; + ETag: string; +} + +async function uploadStreamUsingMultipart( + fileLocalID: number, + dataStream: DataStream, + makeProgessTracker: MakeProgressTracker, + isCFUploadProxyDisabled: boolean, + abortIfCancelled: () => void, +) { + const uploadPartCount = Math.ceil( + dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + ); + const multipartUploadURLs = + await uploadService.fetchMultipartUploadURLs(uploadPartCount); + + const { stream } = dataStream; + + const streamReader = stream.getReader(); + const percentPerPart = + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; + const partEtags: PartEtag[] = []; + for (const [ + index, + fileUploadURL, + ] of multipartUploadURLs.partURLs.entries()) { + abortIfCancelled(); + + const uploadChunk = await combineChunksToFormUploadPart(streamReader); + const progressTracker = makeProgessTracker( + fileLocalID, + percentPerPart, + index, + ); + let eTag = null; + if (!isCFUploadProxyDisabled) { + eTag = await UploadHttpClient.putFilePartV2( + fileUploadURL, + uploadChunk, + progressTracker, + ); + } else { + eTag = await UploadHttpClient.putFilePart( + fileUploadURL, + uploadChunk, + progressTracker, + ); + } + partEtags.push({ PartNumber: index + 1, ETag: eTag }); + } + const { done } = await streamReader.read(); + if (!done) { + throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); + } + + const completeURL = multipartUploadURLs.completeURL; + const cBody = convert.js2xml( + { CompleteMultipartUpload: { Part: partEtags } }, + { compact: true, ignoreComment: true, spaces: 4 }, + ); + if (!isCFUploadProxyDisabled) { + await UploadHttpClient.completeMultipartUploadV2(completeURL, cBody); + } else { + await UploadHttpClient.completeMultipartUpload(completeURL, cBody); + } + + return multipartUploadURLs.objectKey; +} + +async function combineChunksToFormUploadPart( + streamReader: ReadableStreamDefaultReader, +) { + const combinedChunks = []; + for (let i = 0; i < FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { + const { done, value: chunk } = await streamReader.read(); + if (done) { + break; + } + for (let index = 0; index < chunk.length; index++) { + combinedChunks.push(chunk[index]); } } + return Uint8Array.from(combinedChunks); } diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 29a1cffef2..5918bb9dde 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,4 +1,4 @@ -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { City } from "services/locationSearchService"; import { LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 78b46670c6..bdcef330b2 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -1,28 +1,19 @@ +import { FILE_TYPE } from "@/media/file"; +import type { ElectronFile } from "@/next/types/file"; import { B64EncryptionResult, LocalFileAttributes, } from "@ente/shared/crypto/types"; -import { FILE_TYPE } from "constants/file"; +import type { DataStream } from "@ente/shared/utils/data-stream"; import { Collection } from "types/collection"; import { FilePublicMagicMetadata, - FilePublicMagicMetadataProps, MetadataFileAttributes, S3FileAttributes, } from "types/file"; import { EncryptedMagicMetadata } from "types/magicMetadata"; -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - -export function isDataStream(object: any): object is DataStream { - return "stream" in object; -} - -export type Logger = (message: string) => void; - +/** Information about the file that never changes post upload. */ export interface Metadata { /** * The file name. @@ -49,13 +40,6 @@ export interface Location { longitude: number; } -export interface ParsedMetadataJSON { - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; -} - export interface MultipartUploadURLs { objectKey: string; partURLs: string[]; @@ -70,24 +54,6 @@ export interface FileTypeInfo { videoType?: string; } -/* - * ElectronFile is a custom interface that is used to represent - * any file on disk as a File-like object in the Electron desktop app. - * - * This was added to support the auto-resuming of failed uploads - * which needed absolute paths to the files which the - * normal File interface does not provide. - */ -export interface ElectronFile { - name: string; - path: string; - size: number; - lastModified: number; - stream: () => Promise>; - blob: () => Promise; - arrayBuffer: () => Promise; -} - export interface UploadAsset { isLivePhoto?: boolean; file?: File | ElectronFile; @@ -107,13 +73,13 @@ export interface FileWithCollection extends UploadAsset { export interface UploadAsset2 { isLivePhoto?: boolean; - file?: File | ElectronFile | string; + file?: File | string; livePhotoAssets?: LivePhotoAssets2; } export interface LivePhotoAssets2 { - image: File | ElectronFile | string; - video: File | ElectronFile | string; + image: File | string; + video: File | string; } export interface FileWithCollection2 extends UploadAsset2 { @@ -122,8 +88,6 @@ export interface FileWithCollection2 extends UploadAsset2 { collectionID?: number; } -export type ParsedMetadataJSONMap = Map; - export interface UploadURL { url: string; objectKey: string; @@ -131,7 +95,12 @@ export interface UploadURL { export interface FileInMemory { filedata: Uint8Array | DataStream; + /** The JPEG data of the generated thumbnail */ thumbnail: Uint8Array; + /** + * `true` if this is a fallback (all black) thumbnail we're returning since + * thumbnail generation failed for some reason. + */ hasStaticThumbnail: boolean; } @@ -146,6 +115,7 @@ export interface EncryptedFile { file: ProcessedFile; fileKey: B64EncryptionResult; } + export interface ProcessedFile { file: LocalFileAttributes; thumbnail: LocalFileAttributes; @@ -178,8 +148,3 @@ export interface PublicUploadProps { passwordToken: string; accessedThroughSharedURL: boolean; } - -export interface ExtractMetadataResult { - metadata: Metadata; - publicMagicMetadata: FilePublicMagicMetadataProps; -} diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index a6275f254f..fa681c2cde 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; @@ -6,15 +7,6 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils"; -import { - FILE_TYPE, - RAW_FORMATS, - SUPPORTED_RAW_FORMATS, - TYPE_HEIC, - TYPE_HEIF, - TYPE_JPEG, - TYPE_JPG, -} from "constants/file"; import { t } from "i18next"; import isElectron from "is-electron"; import { moveToHiddenCollection } from "services/collectionService"; @@ -48,6 +40,38 @@ import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; import { safeFileName } from "utils/native-fs"; import { writeStream } from "utils/native-stream"; +const TYPE_HEIC = "heic"; +const TYPE_HEIF = "heif"; +const TYPE_JPEG = "jpeg"; +const TYPE_JPG = "jpg"; + +const RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "raf", + "nef", + "psd", + "dng", + "tif", +]; + +const SUPPORTED_RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "nef", + "psd", + "dng", + "tif", +]; + export enum FILE_OPS_TYPE { DOWNLOAD, FIX_TIME, @@ -62,8 +86,10 @@ class ModuleState { /** * This will be set to true if we get an error from the Node.js side of our * desktop app telling us that native JPEG conversion is not available for - * the current OS/arch combination. That way, we can stop pestering it again - * and again (saving an IPC round-trip). + * the current OS/arch combination. + * + * That way, we can stop pestering it again and again (saving an IPC + * round-trip). * * Note the double negative when it is used. */ @@ -292,14 +318,12 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { return imageBlob; } - let jpegBlob: Blob | undefined; - const available = !moduleState.isNativeJPEGConversionNotAvailable; if (isElectron() && available && isSupportedRawFormat(exactType)) { // If we're running in our desktop app, see if our Node.js layer can // convert this into a JPEG using native tools for us. try { - jpegBlob = await nativeConvertToJPEG(fileName, imageBlob); + return await nativeConvertToJPEG(imageBlob); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { moduleState.isNativeJPEGConversionNotAvailable = true; @@ -309,12 +333,12 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } } - if (!jpegBlob && isFileHEIC(exactType)) { + if (isFileHEIC(exactType)) { // If it is an HEIC file, use our web HEIC converter. - jpegBlob = await heicToJPEG(imageBlob); + return await heicToJPEG(imageBlob); } - return jpegBlob; + return undefined; } catch (e) { log.error( `Failed to get renderable image for ${JSON.stringify(fileTypeInfo ?? fileName)}`, @@ -324,7 +348,7 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } }; -const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => { +const nativeConvertToJPEG = async (imageBlob: Blob) => { const startTime = Date.now(); const imageData = new Uint8Array(await imageBlob.arrayBuffer()); const electron = globalThis.electron; @@ -332,8 +356,8 @@ const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => { // the main thread since workers don't have access to the `window` (and // thus, to the `window.electron`) object. const jpegData = electron - ? await electron.convertToJPEG(fileName, imageData) - : await workerBridge.convertToJPEG(fileName, imageData); + ? await electron.convertToJPEG(imageData) + : await workerBridge.convertToJPEG(imageData); log.debug(() => `Native JPEG conversion took ${Date.now() - startTime} ms`); return new Blob([jpegData]); }; @@ -441,6 +465,18 @@ export function isSharedFile(user: User, file: EnteFile) { return file.ownerID !== user.id; } +/** + * [Note: File name for local EnteFile objects] + * + * The title property in a file's metadata is the original file's name. The + * metadata of a file cannot be edited. So if later on the file's name is + * changed, then the edit is stored in the `editedName` property of the public + * metadata of the file. + * + * This function merges these edits onto the file object that we use locally. + * Effectively, post this step, the file's metadata.title can be used in lieu of + * its filename. + */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { return files.map((file) => { if (file.pubMagicMetadata?.data.editedTime) { @@ -646,7 +682,11 @@ async function downloadFileDesktop( fs.exists, ); const imageStream = generateStreamFromArrayBuffer(imageData); - await writeStream(`${downloadDir}/${imageExportName}`, imageStream); + await writeStream( + electron, + `${downloadDir}/${imageExportName}`, + imageStream, + ); try { const videoExportName = await safeFileName( downloadDir, @@ -654,7 +694,11 @@ async function downloadFileDesktop( fs.exists, ); const videoStream = generateStreamFromArrayBuffer(videoData); - await writeStream(`${downloadDir}/${videoExportName}`, videoStream); + await writeStream( + electron, + `${downloadDir}/${videoExportName}`, + videoStream, + ); } catch (e) { await fs.rm(`${downloadDir}/${imageExportName}`); throw e; @@ -665,7 +709,11 @@ async function downloadFileDesktop( file.metadata.title, fs.exists, ); - await writeStream(`${downloadDir}/${fileExportName}`, updatedStream); + await writeStream( + electron, + `${downloadDir}/${fileExportName}`, + updatedStream, + ); } } @@ -690,14 +738,14 @@ export const getUserOwnedFiles = (files: EnteFile[]) => { }; // doesn't work on firefox -export const copyFileToClipboard = async (fileUrl: string) => { +export const copyFileToClipboard = async (fileURL: string) => { const canvas = document.createElement("canvas"); const canvasCTX = canvas.getContext("2d"); const image = new Image(); const blobPromise = new Promise((resolve, reject) => { try { - image.setAttribute("src", fileUrl); + image.setAttribute("src", fileURL); image.onload = () => { canvas.width = image.width; canvas.height = image.height; diff --git a/web/apps/photos/src/utils/file/livePhoto.ts b/web/apps/photos/src/utils/file/livePhoto.ts deleted file mode 100644 index 7d687217ce..0000000000 --- a/web/apps/photos/src/utils/file/livePhoto.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { FILE_TYPE } from "constants/file"; -import { getFileExtension } from "utils/file"; - -const IMAGE_EXTENSIONS = [ - "heic", - "heif", - "jpeg", - "jpg", - "png", - "gif", - "bmp", - "tiff", - "webp", -]; - -const VIDEO_EXTENSIONS = [ - "mov", - "mp4", - "m4v", - "avi", - "wmv", - "flv", - "mkv", - "webm", - "3gp", - "3g2", - "avi", - "ogv", - "mpg", - "mp", -]; - -export function getFileTypeFromExtensionForLivePhotoClustering( - filename: string, -) { - const extension = getFileExtension(filename)?.toLowerCase(); - if (IMAGE_EXTENSIONS.includes(extension)) { - return FILE_TYPE.IMAGE; - } else if (VIDEO_EXTENSIONS.includes(extension)) { - return FILE_TYPE.VIDEO; - } -} diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index a89bccc4ca..4e9b795ec1 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import PQueue from "p-queue"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index 7dba1acf9c..a9a76b41be 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -4,17 +4,71 @@ * NOTE: These functions only work when we're running in our desktop app. */ +import type { Electron } from "@/next/types/ipc"; + /** - * Write the given stream to a file on the local machine. + * Stream the given file from the user's local filesystem. * - * **This only works when we're running in our desktop app**. It uses the + * This only works when we're running in our desktop app since it uses the * "stream://" protocol handler exposed by our custom code in the Node.js layer. * See: [Note: IPC streams]. * + * To avoid accidentally invoking it in a non-desktop app context, it requires + * the {@link Electron} object as a parameter (even though it doesn't use it). + * + * @param path The path on the file on the user's local filesystem whose + * contents we want to stream. + * + * @return A ({@link Response}, size) tuple. + * + * * The response contains the contents of the file. In particular, the `body` + * {@link ReadableStream} property of this response can be used to read the + * files contents in a streaming manner. + * + * * The size is the size of the file that we'll be reading from disk. + */ +export const readStream = async ( + _: Electron, + path: string, +): Promise<{ response: Response; size: number }> => { + const req = new Request(`stream://read${path}`, { + method: "GET", + }); + + const res = await fetch(req); + if (!res.ok) + throw new Error( + `Failed to read stream from ${path}: HTTP ${res.status}`, + ); + + const size = +res.headers["Content-Length"]; + if (isNaN(size)) + throw new Error( + `Got a numeric Content-Length when reading a stream. The response was ${res}`, + ); + + return { response: res, size }; +}; + +/** + * Write the given stream to a file on the local machine. + * + * This only works when we're running in our desktop app since it uses the + * "stream://" protocol handler exposed by our custom code in the Node.js layer. + * See: [Note: IPC streams]. + * + * To avoid accidentally invoking it in a non-desktop app context, it requires + * the {@link Electron} object as a parameter (even though it doesn't use it). + * * @param path The path on the local machine where to write the file to. + * * @param stream The stream which should be written into the file. - * */ -export const writeStream = async (path: string, stream: ReadableStream) => { + */ +export const writeStream = async ( + _: Electron, + path: string, + stream: ReadableStream, +) => { // TODO(MR): This doesn't currently work. // // Not sure what I'm doing wrong here; I've opened an issue upstream diff --git a/web/apps/photos/src/utils/photoFrame/index.ts b/web/apps/photos/src/utils/photoFrame/index.ts index faf0679e7f..ccd47b604d 100644 --- a/web/apps/photos/src/utils/photoFrame/index.ts +++ b/web/apps/photos/src/utils/photoFrame/index.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import { LivePhotoSourceURL, SourceURLs } from "services/download"; import { EnteFile } from "types/file"; import { SetSelectedState } from "types/gallery"; diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 7d082166cf..091026e7be 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,89 +1,18 @@ import { basename, dirname } from "@/next/file"; -import { FILE_TYPE } from "constants/file"; +import { ElectronFile } from "@/next/types/file"; import { PICKED_UPLOAD_TYPE } from "constants/upload"; import isElectron from "is-electron"; import { exportMetadataDirectoryName } from "services/export"; -import { EnteFile } from "types/file"; import { - ElectronFile, FileWithCollection, Metadata, type FileWithCollection2, } from "types/upload"; const TYPE_JSON = "json"; -const DEDUPE_COLLECTION = new Set(["icloud library", "icloudlibrary"]); -export function findMatchingExistingFiles( - existingFiles: EnteFile[], - newFileMetadata: Metadata, -): EnteFile[] { - const matchingFiles: EnteFile[] = []; - for (const existingFile of existingFiles) { - if (areFilesSame(existingFile.metadata, newFileMetadata)) { - matchingFiles.push(existingFile); - } - } - return matchingFiles; -} - -export function shouldDedupeAcrossCollection(collectionName: string): boolean { - // using set to avoid unnecessary regex for removing spaces for each upload - return DEDUPE_COLLECTION.has(collectionName.toLocaleLowerCase()); -} - -export function areFilesSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if (hasFileHash(existingFile) && hasFileHash(newFile)) { - return areFilesWithFileHashSame(existingFile, newFile); - } else { - /* - * The maximum difference in the creation/modification times of two similar files is set to 1 second. - * This is because while uploading files in the web - browsers and users could have set reduced - * precision of file times to prevent timing attacks and fingerprinting. - * Context: https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision - */ - const oneSecond = 1e6; - if ( - existingFile.fileType === newFile.fileType && - Math.abs(existingFile.creationTime - newFile.creationTime) < - oneSecond && - Math.abs(existingFile.modificationTime - newFile.modificationTime) < - oneSecond && - existingFile.title === newFile.title - ) { - return true; - } else { - return false; - } - } -} - -export function hasFileHash(file: Metadata) { - return file.hash || (file.imageHash && file.videoHash); -} - -export function areFilesWithFileHashSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if ( - existingFile.fileType !== newFile.fileType || - existingFile.title !== newFile.title - ) { - return false; - } - if (existingFile.fileType === FILE_TYPE.LIVE_PHOTO) { - return ( - existingFile.imageHash === newFile.imageHash && - existingFile.videoHash === newFile.videoHash - ); - } else { - return existingFile.hash === newFile.hash; - } -} +export const hasFileHash = (file: Metadata) => + file.hash || (file.imageHash && file.videoHash); export function segregateMetadataAndMediaFiles( filesWithCollectionToUpload: FileWithCollection[], @@ -101,23 +30,6 @@ export function segregateMetadataAndMediaFiles( return { mediaFiles, metadataJSONFiles }; } -export function segregateMetadataAndMediaFiles2( - filesWithCollectionToUpload: FileWithCollection2[], -) { - const metadataJSONFiles: FileWithCollection2[] = []; - const mediaFiles: FileWithCollection2[] = []; - filesWithCollectionToUpload.forEach((fileWithCollection) => { - const file = fileWithCollection.file; - const s = typeof file == "string" ? file : file.name; - if (s.toLowerCase().endsWith(TYPE_JSON)) { - metadataJSONFiles.push(fileWithCollection); - } else { - mediaFiles.push(fileWithCollection); - } - }); - return { mediaFiles, metadataJSONFiles }; -} - export function areFileWithCollectionsSame( firstFile: FileWithCollection2, secondFile: FileWithCollection2, diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index 8403c3f6c8..03893efba6 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -1,8 +1,6 @@ -import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { withTimeout } from "@ente/shared/utils"; import QueueProcessor from "@ente/shared/utils/queueProcessor"; -import { generateTempName } from "@ente/shared/utils/temp"; import { expose } from "comlink"; import { ffmpegPathPlaceholder, @@ -10,108 +8,106 @@ import { outputPathPlaceholder, } from "constants/ffmpeg"; import { FFmpeg, createFFmpeg } from "ffmpeg-wasm"; -import { getUint8ArrayView } from "services/readerService"; export class DedicatedFFmpegWorker { - private wasmFFmpeg: WasmFFmpeg; - - constructor() { - this.wasmFFmpeg = new WasmFFmpeg(); - } - - /** - * Execute a ffmpeg {@link command}. - * - * This is a sibling of {@link ffmpegExec} in ipc.ts exposed by the desktop - * app. See [Note: ffmpeg in Electron]. - */ - run(cmd, inputFile, outputFileName, timeoutMS) { - return this.wasmFFmpeg.run(cmd, inputFile, outputFileName, timeoutMS); - } -} - -expose(DedicatedFFmpegWorker, self); - -export class WasmFFmpeg { private ffmpeg: FFmpeg; - private ready: Promise = null; - private ffmpegTaskQueue = new QueueProcessor(); + private ffmpegTaskQueue = new QueueProcessor(); constructor() { this.ffmpeg = createFFmpeg({ corePath: "/js/ffmpeg/ffmpeg-core.js", mt: false, }); - - this.ready = this.init(); } - private async init() { - if (!this.ffmpeg.isLoaded()) { - await this.ffmpeg.load(); - } - } + /** + * Execute a FFmpeg {@link command} on {@link blob}. + * + * This is a sibling of {@link ffmpegExec} exposed by the desktop app in + * `ipc.ts`. See [Note: FFmpeg in Electron]. + */ + async exec( + command: string[], + blob: Blob, + outputFileExtension: string, + timeoutMs, + ): Promise { + if (!this.ffmpeg.isLoaded()) await this.ffmpeg.load(); + + const go = () => + ffmpegExec(this.ffmpeg, command, outputFileExtension, blob); - async run( - cmd: string[], - inputFile: File, - outputFileName: string, - timeoutMS, - ) { - const exec = () => this.execute(cmd, inputFile, outputFileName); const request = this.ffmpegTaskQueue.queueUpRequest(() => - timeoutMS ? withTimeout(exec(), timeoutMS) : exec(), + timeoutMs ? withTimeout(go(), timeoutMs) : go(), ); + return await request.promise; } +} - private async execute( - cmd: string[], - inputFile: File, - outputFileName: string, - ) { - let tempInputFilePath: string; - let tempOutputFilePath: string; +expose(DedicatedFFmpegWorker, self); + +const ffmpegExec = async ( + ffmpeg: FFmpeg, + command: string[], + outputFileExtension: string, + blob: Blob, +) => { + const inputPath = randomPrefix(); + const outputSuffix = outputFileExtension ? "." + outputFileExtension : ""; + const outputPath = randomPrefix() + outputSuffix; + + const cmd = substitutePlaceholders(command, inputPath, outputPath); + + const inputData = new Uint8Array(await blob.arrayBuffer()); + + try { + ffmpeg.FS("writeFile", inputPath, inputData); + + log.debug(() => `[wasm] ffmpeg ${cmd.join(" ")}`); + await ffmpeg.run(...cmd); + + return ffmpeg.FS("readFile", outputPath); + } finally { try { - await this.ready; - const [, extension] = nameAndExtension(inputFile.name); - const tempNameSuffix = extension ? `input.${extension}` : "input"; - tempInputFilePath = `${generateTempName(10, tempNameSuffix)}`; - this.ffmpeg.FS( - "writeFile", - tempInputFilePath, - await getUint8ArrayView(inputFile), - ); - tempOutputFilePath = `${generateTempName(10, outputFileName)}`; - - cmd = cmd.map((cmdPart) => { - if (cmdPart === ffmpegPathPlaceholder) { - return ""; - } else if (cmdPart === inputPathPlaceholder) { - return tempInputFilePath; - } else if (cmdPart === outputPathPlaceholder) { - return tempOutputFilePath; - } else { - return cmdPart; - } - }); - log.info(`${cmd}`); - await this.ffmpeg.run(...cmd); - return new File( - [this.ffmpeg.FS("readFile", tempOutputFilePath)], - outputFileName, - ); - } finally { - try { - this.ffmpeg.FS("unlink", tempInputFilePath); - } catch (e) { - log.error("unlink input file failed", e); - } - try { - this.ffmpeg.FS("unlink", tempOutputFilePath); - } catch (e) { - log.error("unlink output file failed", e); - } + ffmpeg.FS("unlink", inputPath); + } catch (e) { + log.error(`Failed to remove input ${inputPath}`, e); + } + try { + ffmpeg.FS("unlink", outputPath); + } catch (e) { + log.error(`Failed to remove output ${outputPath}`, e); } } -} +}; + +/** Generate a random string suitable for being used as a file name prefix */ +const randomPrefix = () => { + const alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + let result = ""; + for (let i = 0; i < 10; i++) + result += alphabet[Math.floor(Math.random() * alphabet.length)]; + return result; +}; + +const substitutePlaceholders = ( + command: string[], + inputFilePath: string, + outputFilePath: string, +) => + command + .map((segment) => { + if (segment == ffmpegPathPlaceholder) { + return undefined; + } else if (segment == inputPathPlaceholder) { + return inputFilePath; + } else if (segment == outputPathPlaceholder) { + return outputFilePath; + } else { + return segment; + } + }) + .filter((c) => !!c); diff --git a/web/apps/photos/tests/upload.test.ts b/web/apps/photos/tests/upload.test.ts index 6e58cf0c2d..3fc7e8541e 100644 --- a/web/apps/photos/tests/upload.test.ts +++ b/web/apps/photos/tests/upload.test.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import { tryToParseDateTime } from "@ente/shared/time"; -import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import { getLocalFiles } from "services/fileService"; import { @@ -7,7 +7,7 @@ import { getClippedMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForJSON, -} from "services/upload/metadataService"; +} from "services/upload/takeout"; import { getUserDetailsV2 } from "services/userService"; import { groupFilesBasedOnCollectionID } from "utils/file"; diff --git a/web/apps/photos/tests/zip-file-reading.test.ts b/web/apps/photos/tests/zip-file-reading.test.ts index 07d70f067d..ea7511d0b4 100644 --- a/web/apps/photos/tests/zip-file-reading.test.ts +++ b/web/apps/photos/tests/zip-file-reading.test.ts @@ -1,7 +1,7 @@ import { getFileNameSize } from "@/next/file"; +import type { DataStream } from "@ente/shared/utils/data-stream"; import { FILE_READER_CHUNK_SIZE, PICKED_UPLOAD_TYPE } from "constants/upload"; import { getElectronFileStream, getFileStream } from "services/readerService"; -import { DataStream } from "types/upload"; import { getImportSuggestion } from "utils/upload"; // This was for used to verify that converting from the browser readable stream diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts new file mode 100644 index 0000000000..8917db8025 --- /dev/null +++ b/web/packages/media/file.ts @@ -0,0 +1,6 @@ +export enum FILE_TYPE { + IMAGE, + VIDEO, + LIVE_PHOTO, + OTHERS, +} diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index 16143ca138..c4a6314673 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -1,5 +1,52 @@ import { fileNameFromComponents, nameAndExtension } from "@/next/file"; import JSZip from "jszip"; +import { FILE_TYPE } from "./file"; + +const potentialImageExtensions = [ + "heic", + "heif", + "jpeg", + "jpg", + "png", + "gif", + "bmp", + "tiff", + "webp", +]; + +const potentialVideoExtensions = [ + "mov", + "mp4", + "m4v", + "avi", + "wmv", + "flv", + "mkv", + "webm", + "3gp", + "3g2", + "avi", + "ogv", + "mpg", + "mp", +]; + +/** + * Use the file extension of the given {@link fileName} to deduce if is is + * potentially the image or the video part of a Live Photo. + */ +export const potentialFileTypeFromExtension = ( + fileName: string, +): FILE_TYPE | undefined => { + let [, ext] = nameAndExtension(fileName); + if (!ext) return undefined; + + ext = ext.toLowerCase(); + + if (potentialImageExtensions.includes(ext)) return FILE_TYPE.IMAGE; + else if (potentialVideoExtensions.includes(ext)) return FILE_TYPE.VIDEO; + else return undefined; +}; /** * An in-memory representation of a live photo. diff --git a/web/packages/next/blob-cache.ts b/web/packages/next/blob-cache.ts index 8789a50786..0e092fed61 100644 --- a/web/packages/next/blob-cache.ts +++ b/web/packages/next/blob-cache.ts @@ -113,6 +113,10 @@ export const openCache = async ( * * await blob.arrayBuffer() * + * To convert from a Blob to Uint8Array, chain the two steps + * + * new Uint8Array(await blob.arrayBuffer()) + * * To convert from an ArrayBuffer or Uint8Array to Blob * * new Blob([arrayBuffer, andOrAnyArray, andOrstring]) diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index 83b20f2ec5..02f936a18c 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -66,6 +66,13 @@ export const dirname = (path: string) => { return pathComponents.join("/"); }; +/** + * Return a short description of the given {@link fileOrPath} suitable for + * helping identify it in log messages. + */ +export const fopLabel = (fileOrPath: File | string) => + fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath; + export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`; } diff --git a/web/packages/next/types/file.ts b/web/packages/next/types/file.ts index dc8a148e93..75641e3a27 100644 --- a/web/packages/next/types/file.ts +++ b/web/packages/next/types/file.ts @@ -1,8 +1,3 @@ -export enum UPLOAD_STRATEGY { - SINGLE_COLLECTION, - COLLECTION_PER_FOLDER, -} - /* * ElectronFile is a custom interface that is used to represent * any file on disk as a File-like object in the Electron desktop app. @@ -21,11 +16,6 @@ export interface ElectronFile { arrayBuffer: () => Promise; } -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - export interface EventQueueItem { type: "upload" | "trash"; folderPath: string; diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index d87b8e830d..cdc3597ec9 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -204,14 +204,11 @@ export interface Electron { * yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param fileName The name of the file whose data we're being given. * @param imageData The raw image data (the contents of the image file). + * * @returns JPEG data of the converted image. */ - convertToJPEG: ( - fileName: string, - imageData: Uint8Array, - ) => Promise; + convertToJPEG: (imageData: Uint8Array) => Promise; /** * Generate a JPEG thumbnail for the given image. @@ -224,24 +221,26 @@ export interface Electron { * not yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param inputFile The file whose thumbnail we want. + * @param dataOrPath The raw image data (the contents of the image file), or + * the path to the image file, whose thumbnail we want to generate. * @param maxDimension The maximum width or height of the generated * thumbnail. * @param maxSize Maximum size (in bytes) of the generated thumbnail. + * * @returns JPEG data of the generated thumbnail. */ generateImageThumbnail: ( - inputFile: File | ElectronFile, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ) => Promise; /** - * Execute a ffmpeg {@link command}. + * Execute a FFmpeg {@link command} on the given {@link dataOrPath}. * - * This executes the command using the ffmpeg executable we bundle with our - * desktop app. There is also a ffmpeg wasm implementation that we use when - * running on the web, it also has a sibling function with the same + * This executes the command using a FFmpeg executable we bundle with our + * desktop app. We also have a wasm FFmpeg wasm implementation that we use + * when running on the web, which has a sibling function with the same * parameters. See [Note: ffmpeg in Electron]. * * @param command An array of strings, each representing one positional @@ -250,25 +249,27 @@ export interface Electron { * (respectively {@link inputPathPlaceholder}, * {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}). * - * @param inputDataOrPath The bytes of the input file, or the path to the - * input file on the user's local disk. In both cases, the data gets - * serialized to a temporary file, and then that path gets substituted in - * the ffmpeg {@link command} by {@link inputPathPlaceholder}. + * @param dataOrPath The bytes of the input file, or the path to the input + * file on the user's local disk. In both cases, the data gets serialized to + * a temporary file, and then that path gets substituted in the FFmpeg + * {@link command} in lieu of {@link inputPathPlaceholder}. * - * @param outputFileName The name of the file we instruct ffmpeg to produce - * when giving it the given {@link command}. The contents of this file get - * returned as the result. + * @param outputFileExtension The extension (without the dot, e.g. "jpeg") + * to use for the output file that we ask FFmpeg to create in + * {@param command}. While this file will eventually get deleted, and we'll + * just return its contents, for some FFmpeg command the extension matters + * (e.g. conversion to a JPEG fails if the extension is arbitrary). * * @param timeoutMS If non-zero, then abort and throw a timeout error if the * ffmpeg command takes more than the given number of milliseconds. * * @returns The contents of the output file produced by the ffmpeg command - * at {@link outputFileName}. + * (specified as {@link outputPathPlaceholder} in {@link command}). */ ffmpegExec: ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ) => Promise; diff --git a/web/packages/next/worker/comlink-worker.ts b/web/packages/next/worker/comlink-worker.ts index 7bae126a40..5929e5361b 100644 --- a/web/packages/next/worker/comlink-worker.ts +++ b/web/packages/next/worker/comlink-worker.ts @@ -44,8 +44,8 @@ const workerBridge = { logToDisk, // Needed by ML worker getAuthToken: () => ensureLocalUser().then((user) => user.token), - convertToJPEG: (fileName: string, imageData: Uint8Array) => - ensureElectron().convertToJPEG(fileName, imageData), + convertToJPEG: (imageData: Uint8Array) => + ensureElectron().convertToJPEG(imageData), detectFaces: (input: Float32Array) => ensureElectron().detectFaces(input), faceEmbedding: (input: Float32Array) => ensureElectron().faceEmbedding(input), diff --git a/web/packages/shared/crypto/types.ts b/web/packages/shared/crypto/types.ts index 4cf4c56b1f..47bfa8b2ce 100644 --- a/web/packages/shared/crypto/types.ts +++ b/web/packages/shared/crypto/types.ts @@ -1,4 +1,4 @@ -import { DataStream } from "@/next/types/file"; +import type { DataStream } from "../utils/data-stream"; export interface LocalFileAttributes< T extends string | Uint8Array | DataStream, diff --git a/web/packages/shared/error/index.ts b/web/packages/shared/error/index.ts index e9c9270b8e..cf01a38435 100644 --- a/web/packages/shared/error/index.ts +++ b/web/packages/shared/error/index.ts @@ -22,7 +22,6 @@ export function isApiErrorResponse(object: any): object is ApiErrorResponse { } export const CustomError = { - THUMBNAIL_GENERATION_FAILED: "thumbnail generation failed", VIDEO_PLAYBACK_FAILED: "video playback failed", ETAG_MISSING: "no header/etag present in response body", KEY_MISSING: "encrypted key missing from localStorage", @@ -49,7 +48,6 @@ export const CustomError = { SUBSCRIPTION_NEEDED: "subscription not present", NOT_FOUND: "not found ", NO_METADATA: "no metadata", - TOO_LARGE_LIVE_PHOTO_ASSETS: "too large live photo assets", NOT_A_DATE: "not a date", NOT_A_LOCATION: "not a location", FILE_ID_NOT_FOUND: "file with id not found", diff --git a/web/packages/shared/utils/data-stream.ts b/web/packages/shared/utils/data-stream.ts new file mode 100644 index 0000000000..d072dfe7ec --- /dev/null +++ b/web/packages/shared/utils/data-stream.ts @@ -0,0 +1,8 @@ +export interface DataStream { + stream: ReadableStream; + chunkCount: number; +} + +export function isDataStream(object: any): object is DataStream { + return "stream" in object; +} diff --git a/web/packages/shared/utils/temp.ts b/web/packages/shared/utils/temp.ts deleted file mode 100644 index 984f4abb05..0000000000 --- a/web/packages/shared/utils/temp.ts +++ /dev/null @@ -1,14 +0,0 @@ -const CHARACTERS = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - -export function generateTempName(length: number, suffix: string) { - let tempName = ""; - - const charactersLength = CHARACTERS.length; - for (let i = 0; i < length; i++) { - tempName += CHARACTERS.charAt( - Math.floor(Math.random() * charactersLength), - ); - } - return `${tempName}-${suffix}`; -}