From 37f1a7b6303d162b73fa915ecad6af9b3e793dc6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 11:25:14 +0530 Subject: [PATCH 01/82] Mirror the desktop impl --- web/apps/photos/src/worker/ffmpeg.worker.ts | 156 ++++++++++---------- 1 file changed, 76 insertions(+), 80 deletions(-) diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index 8403c3f6c8..4d1c1e88bd 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -13,10 +13,14 @@ import { FFmpeg, createFFmpeg } from "ffmpeg-wasm"; import { getUint8ArrayView } from "services/readerService"; export class DedicatedFFmpegWorker { - private wasmFFmpeg: WasmFFmpeg; + private ffmpeg: FFmpeg; + private ffmpegTaskQueue = new QueueProcessor(); constructor() { - this.wasmFFmpeg = new WasmFFmpeg(); + this.ffmpeg = createFFmpeg({ + corePath: "/js/ffmpeg/ffmpeg-core.js", + mt: false, + }); } /** @@ -25,93 +29,85 @@ export class DedicatedFFmpegWorker { * This is a sibling of {@link ffmpegExec} in ipc.ts exposed by the desktop * app. See [Note: ffmpeg in Electron]. */ - run(cmd, inputFile, outputFileName, timeoutMS) { - return this.wasmFFmpeg.run(cmd, inputFile, outputFileName, timeoutMS); + async execute( + command: string[], + inputFile: File, + outputFileName: string, + timeoutMS, + ) { + if (!this.ffmpeg.isLoaded()) await this.ffmpeg.load(); + + const exec = () => + ffmpegExec(this.ffmpeg, command, inputFile, outputFileName); + + const request = this.ffmpegTaskQueue.queueUpRequest(() => + timeoutMS ? withTimeout(exec(), timeoutMS) : exec(), + ); + + return await request.promise; } } expose(DedicatedFFmpegWorker, self); -export class WasmFFmpeg { - private ffmpeg: FFmpeg; - private ready: Promise = null; - private ffmpegTaskQueue = new QueueProcessor(); +const ffmpegExec = async ( + ffmpeg: FFmpeg, + command: string[], + inputFile: File, + outputFileName: string, +) => { + const [, extension] = nameAndExtension(inputFile.name); + const tempNameSuffix = extension ? `input.${extension}` : "input"; + const tempInputFilePath = `${generateTempName(10, tempNameSuffix)}`; + const tempOutputFilePath = `${generateTempName(10, outputFileName)}`; - constructor() { - this.ffmpeg = createFFmpeg({ - corePath: "/js/ffmpeg/ffmpeg-core.js", - mt: false, - }); + const cmd = substitutePlaceholders( + command, + tempInputFilePath, + tempOutputFilePath, + ); - this.ready = this.init(); - } - - private async init() { - if (!this.ffmpeg.isLoaded()) { - await this.ffmpeg.load(); - } - } - - async run( - cmd: string[], - inputFile: File, - outputFileName: string, - timeoutMS, - ) { - const exec = () => this.execute(cmd, inputFile, outputFileName); - const request = this.ffmpegTaskQueue.queueUpRequest(() => - timeoutMS ? withTimeout(exec(), timeoutMS) : exec(), + try { + ffmpeg.FS( + "writeFile", + tempInputFilePath, + await getUint8ArrayView(inputFile), ); - return await request.promise; - } - private async execute( - cmd: string[], - inputFile: File, - outputFileName: string, - ) { - let tempInputFilePath: string; - let tempOutputFilePath: string; + log.info(`Running ffmpeg (wasm) command ${cmd}`); + await ffmpeg.run(...cmd); + + return new File( + [ffmpeg.FS("readFile", tempOutputFilePath)], + outputFileName, + ); + } finally { try { - await this.ready; - const [, extension] = nameAndExtension(inputFile.name); - const tempNameSuffix = extension ? `input.${extension}` : "input"; - tempInputFilePath = `${generateTempName(10, tempNameSuffix)}`; - this.ffmpeg.FS( - "writeFile", - tempInputFilePath, - await getUint8ArrayView(inputFile), - ); - tempOutputFilePath = `${generateTempName(10, outputFileName)}`; - - cmd = cmd.map((cmdPart) => { - if (cmdPart === ffmpegPathPlaceholder) { - return ""; - } else if (cmdPart === inputPathPlaceholder) { - return tempInputFilePath; - } else if (cmdPart === outputPathPlaceholder) { - return tempOutputFilePath; - } else { - return cmdPart; - } - }); - log.info(`${cmd}`); - await this.ffmpeg.run(...cmd); - return new File( - [this.ffmpeg.FS("readFile", tempOutputFilePath)], - outputFileName, - ); - } finally { - try { - this.ffmpeg.FS("unlink", tempInputFilePath); - } catch (e) { - log.error("unlink input file failed", e); - } - try { - this.ffmpeg.FS("unlink", tempOutputFilePath); - } catch (e) { - log.error("unlink output file failed", e); - } + ffmpeg.FS("unlink", tempInputFilePath); + } catch (e) { + log.error("Failed to remove input ${tempInputFilePath}", e); + } + try { + ffmpeg.FS("unlink", tempOutputFilePath); + } catch (e) { + log.error("Failed to remove output ${tempOutputFilePath}", e); } } -} +}; + +const substitutePlaceholders = ( + command: string[], + inputFilePath: string, + outputFilePath: string, +) => + command.map((segment) => { + if (segment == ffmpegPathPlaceholder) { + return ""; + } else if (segment == inputPathPlaceholder) { + return inputFilePath; + } else if (segment == outputPathPlaceholder) { + return outputFilePath; + } else { + return segment; + } + }); From b1e9f863d75644452df26957965af70489f3985b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 11:49:59 +0530 Subject: [PATCH 02/82] Same --- web/apps/photos/src/services/ffmpeg.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 30ab763232..cce3a35abc 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -12,7 +12,7 @@ import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; /** Called during upload */ export async function generateVideoThumbnail( - file: File | ElectronFile, + fileOrPath: File | ElectronFile | string, ): Promise { let seekTime = 1; while (seekTime >= 0) { @@ -161,20 +161,24 @@ export async function convertToMP4(file: File) { * 10-20x faster than the wasm one currently. See: [Note: ffmpeg in Electron]. */ const ffmpegExec = async ( - cmd: string[], + command: string[], inputFile: File | ElectronFile, - outputFilename: string, + outputFileName: string, timeoutMS: number = 0, ): Promise => { const electron = globalThis.electron; if (electron || false) { - /* TODO(MR): ElectronFile changes */ - // return electron.runFFmpegCmd(cmd, inputFile, outputFilename, timeoutMS); + return electron.ffmpegExec( + command, + inputDataOrPath, + outputFileName, + timeoutMS + ) } else { return workerFactory .instance() .then((worker) => - worker.run(cmd, inputFile, outputFilename, timeoutMS), + worker.execute(command, inputFile, outputFileName, timeoutMS), ); } }; From 2a647e3ddba5b2448c8657f40b3f6feb2fead2de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 11:58:16 +0530 Subject: [PATCH 03/82] Dedup type --- .../photos/src/components/Upload/Uploader.tsx | 7 ++--- web/apps/photos/src/services/ffmpeg.ts | 26 ++++++++++++------- web/apps/photos/src/services/readerService.ts | 2 +- .../src/services/typeDetectionService.ts | 3 ++- .../src/services/upload/metadataService.ts | 2 +- .../photos/src/services/upload/thumbnail.ts | 3 ++- .../src/services/upload/uploadManager.ts | 2 +- .../src/services/upload/uploadService.ts | 2 +- web/apps/photos/src/types/upload/index.ts | 19 +------------- web/apps/photos/src/utils/upload/index.ts | 2 +- 10 files changed, 29 insertions(+), 39 deletions(-) diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index 2ae077daf3..7b4bc4fa34 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -1,5 +1,6 @@ import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import type { CollectionMapping, Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { isPromise } from "@ente/shared/utils"; @@ -32,11 +33,7 @@ import { SetLoading, UploadTypeSelectorIntent, } from "types/gallery"; -import { - ElectronFile, - FileWithCollection, - type FileWithCollection2, -} from "types/upload"; +import { FileWithCollection, type FileWithCollection2 } from "types/upload"; import { InProgressUpload, SegregatedFinishedUploads, diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index cce3a35abc..9f6cec0b6d 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,3 +1,4 @@ +import { ElectronFile } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -7,7 +8,7 @@ import { outputPathPlaceholder, } from "constants/ffmpeg"; import { NULL_LOCATION } from "constants/upload"; -import { ElectronFile, ParsedExtractedMetadata } from "types/upload"; +import { ParsedExtractedMetadata } from "types/upload"; import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; /** Called during upload */ @@ -30,7 +31,8 @@ export async function generateVideoThumbnail( "scale=-1:720", outputPathPlaceholder, ], - file, + /* TODO(MR): ElectronFile changes */ + fileOrPath as File | ElectronFile, "thumb.jpeg", ); } catch (e) { @@ -168,17 +170,23 @@ const ffmpegExec = async ( ): Promise => { const electron = globalThis.electron; if (electron || false) { - return electron.ffmpegExec( - command, - inputDataOrPath, - outputFileName, - timeoutMS - ) + // return electron.ffmpegExec( + // command, + // /* TODO(MR): ElectronFile changes */ + // inputFile as unknown as string, + // outputFileName, + // timeoutMS, + // ); } else { return workerFactory .instance() .then((worker) => - worker.execute(command, inputFile, outputFileName, timeoutMS), + worker.execute( + command, + /* TODO(MR): ElectronFile changes */ inputFile as File, + outputFileName, + timeoutMS, + ), ); } }; diff --git a/web/apps/photos/src/services/readerService.ts b/web/apps/photos/src/services/readerService.ts index e410144cfe..e30710d5ad 100644 --- a/web/apps/photos/src/services/readerService.ts +++ b/web/apps/photos/src/services/readerService.ts @@ -1,6 +1,6 @@ import { convertBytesToHumanReadable } from "@/next/file"; import log from "@/next/log"; -import { ElectronFile } from "types/upload"; +import { ElectronFile } from "@/next/types/file"; export async function getUint8ArrayView( file: Blob | ElectronFile, diff --git a/web/apps/photos/src/services/typeDetectionService.ts b/web/apps/photos/src/services/typeDetectionService.ts index 5ff8f01692..5b53eecbc0 100644 --- a/web/apps/photos/src/services/typeDetectionService.ts +++ b/web/apps/photos/src/services/typeDetectionService.ts @@ -1,4 +1,5 @@ import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { CustomError } from "@ente/shared/error"; import { FILE_TYPE } from "constants/file"; import { @@ -6,7 +7,7 @@ import { WHITELISTED_FILE_FORMATS, } from "constants/upload"; import FileType, { FileTypeResult } from "file-type"; -import { ElectronFile, FileTypeInfo } from "types/upload"; +import { FileTypeInfo } from "types/upload"; import { getFileExtension } from "utils/file"; import { getUint8ArrayView } from "./readerService"; diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index d1c98ff690..367d79bbad 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,6 +1,7 @@ import { ensureElectron } from "@/next/electron"; import { basename, getFileNameSize } from "@/next/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { @@ -17,7 +18,6 @@ import { getFileType } from "services/typeDetectionService"; import { FilePublicMagicMetadataProps } from "types/file"; import { DataStream, - ElectronFile, ExtractMetadataResult, FileTypeInfo, LivePhotoAssets, diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 91b1ea9fb3..82f4486b42 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,12 +1,13 @@ import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { FILE_TYPE } from "constants/file"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; import * as FFmpegService from "services/ffmpeg"; import { heicToJPEG } from "services/heic-convert"; -import { ElectronFile, FileTypeInfo } from "types/upload"; +import { FileTypeInfo } from "types/upload"; import { isFileHEIC } from "utils/file"; import { getUint8ArrayView } from "../readerService"; import { getFileName } from "./uploadService"; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 05a336be59..65ce7d77f2 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,5 +1,6 @@ import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; @@ -18,7 +19,6 @@ import { Collection } from "types/collection"; import { EncryptedEnteFile, EnteFile } from "types/file"; import { SetFiles } from "types/gallery"; import { - ElectronFile, FileWithCollection, ParsedMetadataJSON, ParsedMetadataJSONMap, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 78953bd241..7a118d3030 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -5,6 +5,7 @@ import { getFileNameSize, } from "@/next/file"; import log from "@/next/log"; +import { ElectronFile } from "@/next/types/file"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { B64EncryptionResult, @@ -30,7 +31,6 @@ import { EncryptedMagicMetadata } from "types/magicMetadata"; import { BackupedFile, DataStream, - ElectronFile, EncryptedFile, ExtractMetadataResult, FileInMemory, diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 78b46670c6..175af6824f 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -1,3 +1,4 @@ +import type { ElectronFile } from "@/next/types/file"; import { B64EncryptionResult, LocalFileAttributes, @@ -70,24 +71,6 @@ export interface FileTypeInfo { videoType?: string; } -/* - * ElectronFile is a custom interface that is used to represent - * any file on disk as a File-like object in the Electron desktop app. - * - * This was added to support the auto-resuming of failed uploads - * which needed absolute paths to the files which the - * normal File interface does not provide. - */ -export interface ElectronFile { - name: string; - path: string; - size: number; - lastModified: number; - stream: () => Promise>; - blob: () => Promise; - arrayBuffer: () => Promise; -} - export interface UploadAsset { isLivePhoto?: boolean; file?: File | ElectronFile; diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 7d082166cf..ac05122aa9 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,11 +1,11 @@ import { basename, dirname } from "@/next/file"; +import { ElectronFile } from "@/next/types/file"; import { FILE_TYPE } from "constants/file"; import { PICKED_UPLOAD_TYPE } from "constants/upload"; import isElectron from "is-electron"; import { exportMetadataDirectoryName } from "services/export"; import { EnteFile } from "types/file"; import { - ElectronFile, FileWithCollection, Metadata, type FileWithCollection2, From afca600a69535a642d441b524e9a3120628d0c9b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 14:04:49 +0530 Subject: [PATCH 04/82] New abstraction --- web/packages/next/types/file.ts | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/web/packages/next/types/file.ts b/web/packages/next/types/file.ts index dc8a148e93..17f122f051 100644 --- a/web/packages/next/types/file.ts +++ b/web/packages/next/types/file.ts @@ -1,3 +1,5 @@ +import type { Electron } from "./ipc"; + export enum UPLOAD_STRATEGY { SINGLE_COLLECTION, COLLECTION_PER_FOLDER, @@ -21,6 +23,40 @@ export interface ElectronFile { arrayBuffer: () => Promise; } +/** + * A file path that we obtain from the Node.js layer of our desktop app. + * + * When a user drags and drops or otherwise interactively provides us with a + * file, we get an object that conforms to the [Web File + * API](https://developer.mozilla.org/en-US/docs/Web/API/File). + * + * However, we cannot programmatically create such File objects to arbitrary + * absolute paths on user's local filesystem for security reasons. + * + * This restricts us in cases where the user does want us to, say, watch a + * folder on disk for changes, or auto-resume previously interrupted uploads + * when the app gets restarted. + * + * For such functionality, we defer to our Node.js layer via the + * {@link Electron} object. This IPC communication works with absolute paths of + * disk files or folders, and the native Node.js layer can then perform the + * relevant operations on them. + * + * The {@link DesktopFilePath} interface bundles such a absolute {@link path} + * with an {@link Electron} object that we can later use to, say, read or write + * to that file by using the IPC methods. + * + * This is the same electron instance as `globalThis.electron`, except it is + * non-optional here. Thus we're guaranteed that whatever code is passing us an + * absolute file path is running in the context of our desktop app. + */ +export interface DesktopFilePath { + /** The absolute path to a file or a folder on the local filesystem. */ + path: string; + /** The {@link Electron} instance that we can use to operate on the path. */ + electron: Electron; +} + export interface DataStream { stream: ReadableStream; chunkCount: number; From 875b92ea9140ca7bf36f90b86a3ad0cc95c14c32 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 14:39:48 +0530 Subject: [PATCH 05/82] New interface --- desktop/src/main/services/ffmpeg.ts | 14 +-- web/apps/photos/src/services/ffmpeg.ts | 113 ++++++++++++-------- web/apps/photos/src/worker/ffmpeg.worker.ts | 26 ++--- web/packages/next/blob-cache.ts | 4 + 4 files changed, 88 insertions(+), 69 deletions(-) diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index c49ac67009..5547cf8341 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -9,22 +9,22 @@ const inputPathPlaceholder = "INPUT"; const outputPathPlaceholder = "OUTPUT"; /** - * Run a ffmpeg command + * Run a FFmpeg command * - * [Note: ffmpeg in Electron] + * [Note: FFmpeg in Electron] * - * There is a wasm build of ffmpeg, but that is currently 10-20 times slower + * There is a wasm build of FFmpeg, but that is currently 10-20 times slower * that the native build. That is slow enough to be unusable for our purposes. * https://ffmpegwasm.netlify.app/docs/performance * - * So the alternative is to bundle a ffmpeg binary with our app. e.g. + * So the alternative is to bundle a FFmpeg executable binary with our app. e.g. * * yarn add fluent-ffmpeg ffmpeg-static ffprobe-static * * (we only use ffmpeg-static, the rest are mentioned for completeness' sake). * - * Interestingly, Electron already bundles an ffmpeg library (it comes from the - * ffmpeg fork maintained by Chromium). + * Interestingly, Electron already bundles an binary FFmpeg library (it comes + * from the ffmpeg fork maintained by Chromium). * https://chromium.googlesource.com/chromium/third_party/ffmpeg * https://stackoverflow.com/questions/53963672/what-version-of-ffmpeg-is-bundled-inside-electron * @@ -96,7 +96,7 @@ const substitutePlaceholders = ( /** * Return the path to the `ffmpeg` binary. * - * At runtime, the ffmpeg binary is present in a path like (macOS example): + * At runtime, the FFmpeg binary is present in a path like (macOS example): * `ente.app/Contents/Resources/app.asar.unpacked/node_modules/ffmpeg-static/ffmpeg` */ const ffmpegBinaryPath = () => { diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 9f6cec0b6d..b407d9f4c6 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,4 +1,4 @@ -import { ElectronFile } from "@/next/types/file"; +import { ElectronFile, type DesktopFilePath } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -11,38 +11,42 @@ import { NULL_LOCATION } from "constants/upload"; import { ParsedExtractedMetadata } from "types/upload"; import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; -/** Called during upload */ -export async function generateVideoThumbnail( - fileOrPath: File | ElectronFile | string, -): Promise { - let seekTime = 1; - while (seekTime >= 0) { - try { - return await ffmpegExec( - [ - ffmpegPathPlaceholder, - "-i", - inputPathPlaceholder, - "-ss", - `00:00:0${seekTime}`, - "-vframes", - "1", - "-vf", - "scale=-1:720", - outputPathPlaceholder, - ], - /* TODO(MR): ElectronFile changes */ - fileOrPath as File | ElectronFile, - "thumb.jpeg", - ); - } catch (e) { - if (seekTime === 0) { - throw e; - } - } - seekTime--; +/** + * Generate a thumbnail of the given video using FFmpeg. + * + * This function is called during upload, when we need to generate thumbnails + * for the new files that the user is adding. + * + * @param fileOrPath The input video file or a path to it. + * @returns JPEG data for the generated thumbnail. + */ +export const generateVideoThumbnail = async ( + fileOrPath: File | DesktopFilePath, +): Promise => { + const thumbnailAtTime = (seekTime: number) => + ffmpegExec( + [ + ffmpegPathPlaceholder, + "-i", + inputPathPlaceholder, + "-ss", + `00:00:0${seekTime}`, + "-vframes", + "1", + "-vf", + "scale=-1:720", + outputPathPlaceholder, + ], + fileOrPath, + "thumb.jpeg", + ); + + try { + return await thumbnailAtTime(1); + } catch (e) { + return await thumbnailAtTime(0); } -} +}; /** Called during upload */ export async function extractVideoMetadata(file: File | ElectronFile) { @@ -50,7 +54,7 @@ export async function extractVideoMetadata(file: File | ElectronFile) { // -c [short for codex] copy[(stream_specifier)[ffmpeg.org/ffmpeg.html#Stream-specifiers]] => copies all the stream without re-encoding // -map_metadata [http://ffmpeg.org/ffmpeg.html#Advanced-options search for map_metadata] => copies all stream metadata to the out // -f ffmetadata [https://ffmpeg.org/ffmpeg-formats.html#Metadata-1] => dump metadata from media files into a simple UTF-8-encoded INI-like text file - const metadata = await ffmpegExec( + const metadata = await ffmpegExec2( [ ffmpegPathPlaceholder, "-i", @@ -137,7 +141,7 @@ function parseCreationTime(creationTime: string) { /** Called when viewing a file */ export async function convertToMP4(file: File) { - return await ffmpegExec( + return await ffmpegExec2( [ ffmpegPathPlaceholder, "-i", @@ -153,16 +157,34 @@ export async function convertToMP4(file: File) { } /** - * Run the given ffmpeg command. + * Run the given FFmpeg command. * - * If we're running in the context of our desktop app, use the ffmpeg binary we + * If we're running in the context of our desktop app, use the FFmpeg binary we * bundle with our desktop app to run the command. Otherwise fallback to using - * the wasm ffmpeg we link to from our web app in a web worker. + * the wasm FFmpeg we link to from our web app in a web worker. * - * As a rough ballpark, the native ffmpeg integration in the desktop app is - * 10-20x faster than the wasm one currently. See: [Note: ffmpeg in Electron]. + * As a rough ballpark, the native FFmpeg integration in the desktop app is + * 10-20x faster than the wasm one currently. See: [Note: FFmpeg in Electron]. */ const ffmpegExec = async ( + command: string[], + fileOrPath: File | DesktopFilePath, + outputFileName: string, + timeoutMs: number = 0, +): Promise => { + if (fileOrPath instanceof File) { + return workerFactory + .lazy() + .then((worker) => + worker.exec(command, fileOrPath, outputFileName, timeoutMs), + ); + } else { + const { path, electron } = fileOrPath; + return electron.ffmpegExec(command, path, outputFileName, timeoutMs); + } +}; + +const ffmpegExec2 = async ( command: string[], inputFile: File | ElectronFile, outputFileName: string, @@ -179,7 +201,7 @@ const ffmpegExec = async ( // ); } else { return workerFactory - .instance() + .lazy() .then((worker) => worker.execute( command, @@ -193,14 +215,11 @@ const ffmpegExec = async ( /** Lazily create a singleton instance of our worker */ class WorkerFactory { - private _instance: Promise>; + private instance: Promise>; - async instance() { - if (!this._instance) { - const comlinkWorker = createComlinkWorker(); - this._instance = comlinkWorker.remote; - } - return this._instance; + async lazy() { + if (!this.instance) this.instance = createComlinkWorker().remote; + return this.instance; } } diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index 4d1c1e88bd..cd608f6bc4 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -10,11 +10,10 @@ import { outputPathPlaceholder, } from "constants/ffmpeg"; import { FFmpeg, createFFmpeg } from "ffmpeg-wasm"; -import { getUint8ArrayView } from "services/readerService"; export class DedicatedFFmpegWorker { private ffmpeg: FFmpeg; - private ffmpegTaskQueue = new QueueProcessor(); + private ffmpegTaskQueue = new QueueProcessor(); constructor() { this.ffmpeg = createFFmpeg({ @@ -24,24 +23,24 @@ export class DedicatedFFmpegWorker { } /** - * Execute a ffmpeg {@link command}. + * Execute a FFmpeg {@link command}. * * This is a sibling of {@link ffmpegExec} in ipc.ts exposed by the desktop - * app. See [Note: ffmpeg in Electron]. + * app. See [Note: FFmpeg in Electron]. */ - async execute( + async exec( command: string[], inputFile: File, outputFileName: string, - timeoutMS, - ) { + timeoutMs, + ): Promise { if (!this.ffmpeg.isLoaded()) await this.ffmpeg.load(); - const exec = () => + const go = () => ffmpegExec(this.ffmpeg, command, inputFile, outputFileName); const request = this.ffmpegTaskQueue.queueUpRequest(() => - timeoutMS ? withTimeout(exec(), timeoutMS) : exec(), + timeoutMs ? withTimeout(go(), timeoutMs) : go(), ); return await request.promise; @@ -71,16 +70,13 @@ const ffmpegExec = async ( ffmpeg.FS( "writeFile", tempInputFilePath, - await getUint8ArrayView(inputFile), + new Uint8Array(await inputFile.arrayBuffer()), ); - log.info(`Running ffmpeg (wasm) command ${cmd}`); + log.info(`Running FFmpeg (wasm) command ${cmd}`); await ffmpeg.run(...cmd); - return new File( - [ffmpeg.FS("readFile", tempOutputFilePath)], - outputFileName, - ); + return ffmpeg.FS("readFile", tempOutputFilePath); } finally { try { ffmpeg.FS("unlink", tempInputFilePath); diff --git a/web/packages/next/blob-cache.ts b/web/packages/next/blob-cache.ts index 8789a50786..0e092fed61 100644 --- a/web/packages/next/blob-cache.ts +++ b/web/packages/next/blob-cache.ts @@ -113,6 +113,10 @@ export const openCache = async ( * * await blob.arrayBuffer() * + * To convert from a Blob to Uint8Array, chain the two steps + * + * new Uint8Array(await blob.arrayBuffer()) + * * To convert from an ArrayBuffer or Uint8Array to Blob * * new Blob([arrayBuffer, andOrAnyArray, andOrstring]) From 9038ea79597a854ce054d22c447a0935f74b6fb1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 14:57:38 +0530 Subject: [PATCH 06/82] Dead-endish --- web/apps/photos/src/services/ffmpeg.ts | 3 ++ .../photos/src/services/upload/thumbnail.ts | 36 +++++++------------ web/packages/next/file.ts | 9 ++++- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index b407d9f4c6..6a35f65697 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -42,8 +42,11 @@ export const generateVideoThumbnail = async ( ); try { + // Try generating thumbnail at seekTime 1 second. return await thumbnailAtTime(1); } catch (e) { + // If that fails, try again at the beginning. If even this throws, let + // it fail. return await thumbnailAtTime(0); } }; diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 82f4486b42..68cebc7083 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,6 +1,5 @@ -import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; -import { ElectronFile } from "@/next/types/file"; +import { ElectronFile, type DesktopFilePath } from "@/next/types/file"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { FILE_TYPE } from "constants/file"; @@ -11,6 +10,7 @@ import { FileTypeInfo } from "types/upload"; import { isFileHEIC } from "utils/file"; import { getUint8ArrayView } from "../readerService"; import { getFileName } from "./uploadService"; +import { fopLabel } from "@/next/file"; /** Maximum width or height of the generated thumbnail */ const maxThumbnailDimension = 720; @@ -178,35 +178,23 @@ async function generateImageThumbnailUsingCanvas( return await getUint8ArrayView(thumbnailBlob); } -async function generateVideoThumbnail( - file: File | ElectronFile, - fileTypeInfo: FileTypeInfo, -) { - let thumbnail: Uint8Array; +const generateVideoThumbnail = async (fileOrPath: File | DesktopFilePath) => { try { - log.info( - `ffmpeg generateThumbnail called for ${getFileNameSize(file)}`, - ); - - const thumbnail = await FFmpegService.generateVideoThumbnail(file); - log.info( - `ffmpeg thumbnail successfully generated ${getFileNameSize(file)}`, - ); - return await getUint8ArrayView(thumbnail); + return await FFmpegService.generateVideoThumbnail(fileOrPath); } catch (e) { - log.info( - `ffmpeg thumbnail generated failed ${getFileNameSize( - file, - )} error: ${e.message}`, - ); log.error( - `failed to generate thumbnail using ffmpeg for format ${fileTypeInfo.exactType}`, + `Failed to generate thumbnail using FFmpeg for ${fopLabel(fileOrPath)}`, e, ); - thumbnail = await generateVideoThumbnailUsingCanvas(file); + // If we're on the web, try falling back to using the canvas instead. + if (fileOrPath instanceof File) { + log.info() + } + + return await generateVideoThumbnailUsingCanvas(file); } return thumbnail; -} +}; async function generateVideoThumbnailUsingCanvas(file: File | ElectronFile) { const canvas = document.createElement("canvas"); diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index 83b20f2ec5..4d05225c8e 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -1,4 +1,4 @@ -import type { ElectronFile } from "./types/file"; +import type { DesktopFilePath, ElectronFile } from "./types/file"; /** * The two parts of a file name - the name itself, and an (optional) extension. @@ -66,6 +66,13 @@ export const dirname = (path: string) => { return pathComponents.join("/"); }; +/** + * Return a short description of the given {@link fileOrPath} suitable for + * helping identify it in log messages. + */ +export const fopLabel = (fileOrPath: File | DesktopFilePath) => + fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath.path; + export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`; } From 1c9c6d849a5c7d241e2a1598db458ae81ef0b3b3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 15:30:48 +0530 Subject: [PATCH 07/82] Rework 1 --- .../photos/src/services/upload/thumbnail.ts | 148 ++++++++---------- web/apps/photos/src/utils/file/index.ts | 4 +- web/packages/shared/error/index.ts | 1 - 3 files changed, 66 insertions(+), 87 deletions(-) diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 68cebc7083..93a0248f63 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,7 +1,9 @@ +import { fopLabel } from "@/next/file"; import log from "@/next/log"; import { ElectronFile, type DesktopFilePath } from "@/next/types/file"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; +import { withTimeout } from "@ente/shared/utils"; import { FILE_TYPE } from "constants/file"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; import * as FFmpegService from "services/ffmpeg"; @@ -9,16 +11,11 @@ import { heicToJPEG } from "services/heic-convert"; import { FileTypeInfo } from "types/upload"; import { isFileHEIC } from "utils/file"; import { getUint8ArrayView } from "../readerService"; -import { getFileName } from "./uploadService"; -import { fopLabel } from "@/next/file"; /** Maximum width or height of the generated thumbnail */ const maxThumbnailDimension = 720; /** Maximum size (in bytes) of the generated thumbnail */ const maxThumbnailSize = 100 * 1024; // 100 KB -const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10; -const MIN_QUALITY = 0.5; -const MAX_QUALITY = 0.7; const WAIT_TIME_THUMBNAIL_GENERATION = 30 * 1000; @@ -47,29 +44,34 @@ interface GeneratedThumbnail { } /** - * Generate a JPEG thumbnail for the given {@link file}. + * Generate a JPEG thumbnail for the given image or video data. * * The thumbnail has a smaller file size so that is quick to load. But more * importantly, it uses a universal file format (JPEG in our case) so that the * thumbnail itself can be opened in all clients, even those like the web client * itself that might not yet have support for more exotic formats. + * + * @param blob The data (blob) of the file whose thumbnail we want to generate. + * @param fileTypeInfo The type of the file whose {@link blob} we were given. + * + * @return {@link GeneratedThumbnail}, a thin wrapper for the raw JPEG bytes of + * the generated thumbnail. */ export const generateThumbnail = async ( - file: File | ElectronFile, + blob: Blob, fileTypeInfo: FileTypeInfo, ): Promise => { try { const thumbnail = fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnail(file, fileTypeInfo) - : await generateVideoThumbnail(file, fileTypeInfo); + ? await generateImageThumbnail(blob, fileTypeInfo) + : await generateVideoThumbnail(blob, fileTypeInfo); if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - log.debug(() => `Generated thumbnail for ${getFileName(file)}`); return { thumbnail, hasStaticThumbnail: false }; } catch (e) { log.error( - `Failed to generate thumbnail for ${getFileName(file)} with format ${fileTypeInfo.exactType}`, + `Failed to generate thumbnail for format ${fileTypeInfo.exactType}`, e, ); return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; @@ -84,7 +86,7 @@ const fallbackThumbnail = () => Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); const generateImageThumbnail = async ( - file: File | ElectronFile, + blob: Blob, fileTypeInfo: FileTypeInfo, ) => { let jpegData: Uint8Array | undefined; @@ -107,7 +109,7 @@ const generateImageThumbnail = async ( } if (!jpegData) { - jpegData = await generateImageThumbnailUsingCanvas(file, fileTypeInfo); + jpegData = await generateImageThumbnailUsingCanvas(blob, fileTypeInfo); } return jpegData; }; @@ -128,55 +130,45 @@ const generateImageThumbnailInElectron = async ( return jpegData; }; -async function generateImageThumbnailUsingCanvas( - file: File | ElectronFile, +const generateImageThumbnailUsingCanvas = async ( + blob: Blob, fileTypeInfo: FileTypeInfo, -) { - const canvas = document.createElement("canvas"); - const canvasCTX = canvas.getContext("2d"); - - let imageURL = null; - let timeout = null; - +) => { if (isFileHEIC(fileTypeInfo.exactType)) { - log.debug(() => `Pre-converting ${getFileName(file)} to JPEG`); - const jpegBlob = await heicToJPEG(new Blob([await file.arrayBuffer()])); - file = new File([jpegBlob], file.name); + log.debug(() => `Pre-converting ${fileTypeInfo.exactType} to JPEG`); + blob = await heicToJPEG(blob); } - let image = new Image(); - imageURL = URL.createObjectURL(new Blob([await file.arrayBuffer()])); - await new Promise((resolve, reject) => { - image.setAttribute("src", imageURL); - image.onload = () => { - try { - URL.revokeObjectURL(imageURL); - const { width, height } = scaledThumbnailDimensions( - image.width, - image.height, - maxThumbnailDimension, - ); - canvas.width = width; - canvas.height = height; - canvasCTX.drawImage(image, 0, 0, width, height); - image = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - const err = new Error(CustomError.THUMBNAIL_GENERATION_FAILED, { - cause: e, - }); - reject(err); - } - }; - timeout = setTimeout( - () => reject(new Error("Operation timed out")), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - const thumbnailBlob = await getCompressedThumbnailBlobFromCanvas(canvas); - return await getUint8ArrayView(thumbnailBlob); -} + const canvas = document.createElement("canvas"); + const canvasCtx = canvas.getContext("2d"); + + const imageURL = URL.createObjectURL(blob); + await withTimeout( + new Promise((resolve, reject) => { + const image = new Image(); + image.setAttribute("src", imageURL); + image.onload = () => { + try { + URL.revokeObjectURL(imageURL); + const { width, height } = scaledThumbnailDimensions( + image.width, + image.height, + maxThumbnailDimension, + ); + canvas.width = width; + canvas.height = height; + canvasCtx.drawImage(image, 0, 0, width, height); + resolve(undefined); + } catch (e) { + reject(e); + } + }; + }), + 30 * 1000, + ); + + return await compressedJPEGData(canvas); +}; const generateVideoThumbnail = async (fileOrPath: File | DesktopFilePath) => { try { @@ -188,7 +180,7 @@ const generateVideoThumbnail = async (fileOrPath: File | DesktopFilePath) => { ); // If we're on the web, try falling back to using the canvas instead. if (fileOrPath instanceof File) { - log.info() + log.info(); } return await generateVideoThumbnailUsingCanvas(file); @@ -242,42 +234,30 @@ async function generateVideoThumbnailUsingCanvas(file: File | ElectronFile) { return await getUint8ArrayView(thumbnailBlob); } -async function getCompressedThumbnailBlobFromCanvas(canvas: HTMLCanvasElement) { - let thumbnailBlob: Blob = null; +const compressedJPEGData = async (canvas: HTMLCanvasElement) => { + let blob: Blob; let prevSize = Number.MAX_SAFE_INTEGER; - let quality = MAX_QUALITY; + let quality = 0.7; do { - if (thumbnailBlob) { - prevSize = thumbnailBlob.size; - } - thumbnailBlob = await new Promise((resolve) => { - canvas.toBlob( - function (blob) { - resolve(blob); - }, - "image/jpeg", - quality, - ); + if (blob) prevSize = blob.size; + blob = await new Promise((resolve) => { + canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality); }); - thumbnailBlob = thumbnailBlob ?? new Blob([]); quality -= 0.1; } while ( - quality >= MIN_QUALITY && - thumbnailBlob.size > maxThumbnailSize && - percentageSizeDiff(thumbnailBlob.size, prevSize) >= - MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF + quality >= 0.5 && + blob.size > maxThumbnailSize && + percentageSizeDiff(blob.size, prevSize) >= 10 ); - return thumbnailBlob; -} + return blob; +}; -function percentageSizeDiff( +const percentageSizeDiff = ( newThumbnailSize: number, oldThumbnailSize: number, -) { - return ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; -} +) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; /** * Compute the size of the thumbnail to create for an image with the given diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index a6275f254f..5c715fa482 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -690,14 +690,14 @@ export const getUserOwnedFiles = (files: EnteFile[]) => { }; // doesn't work on firefox -export const copyFileToClipboard = async (fileUrl: string) => { +export const copyFileToClipboard = async (fileURL: string) => { const canvas = document.createElement("canvas"); const canvasCTX = canvas.getContext("2d"); const image = new Image(); const blobPromise = new Promise((resolve, reject) => { try { - image.setAttribute("src", fileUrl); + image.setAttribute("src", fileURL); image.onload = () => { canvas.width = image.width; canvas.height = image.height; diff --git a/web/packages/shared/error/index.ts b/web/packages/shared/error/index.ts index e9c9270b8e..fab3161b21 100644 --- a/web/packages/shared/error/index.ts +++ b/web/packages/shared/error/index.ts @@ -22,7 +22,6 @@ export function isApiErrorResponse(object: any): object is ApiErrorResponse { } export const CustomError = { - THUMBNAIL_GENERATION_FAILED: "thumbnail generation failed", VIDEO_PLAYBACK_FAILED: "video playback failed", ETAG_MISSING: "no header/etag present in response body", KEY_MISSING: "encrypted key missing from localStorage", From bb094743f369ace28ec39d558e230f1d929dc92c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 15:42:35 +0530 Subject: [PATCH 08/82] Rework 2 --- .../photos/src/services/upload/thumbnail.ts | 138 ++++++++---------- 1 file changed, 57 insertions(+), 81 deletions(-) diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 93a0248f63..e10e81f536 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,24 +1,19 @@ -import { fopLabel } from "@/next/file"; import log from "@/next/log"; -import { ElectronFile, type DesktopFilePath } from "@/next/types/file"; +import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; -import { CustomError } from "@ente/shared/error"; import { withTimeout } from "@ente/shared/utils"; import { FILE_TYPE } from "constants/file"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; -import * as FFmpegService from "services/ffmpeg"; +import * as ffmpeg from "services/ffmpeg"; import { heicToJPEG } from "services/heic-convert"; import { FileTypeInfo } from "types/upload"; import { isFileHEIC } from "utils/file"; -import { getUint8ArrayView } from "../readerService"; /** Maximum width or height of the generated thumbnail */ const maxThumbnailDimension = 720; /** Maximum size (in bytes) of the generated thumbnail */ const maxThumbnailSize = 100 * 1024; // 100 KB -const WAIT_TIME_THUMBNAIL_GENERATION = 30 * 1000; - class ModuleState { /** * This will be set to true if we get an error from the Node.js side of our @@ -170,95 +165,51 @@ const generateImageThumbnailUsingCanvas = async ( return await compressedJPEGData(canvas); }; -const generateVideoThumbnail = async (fileOrPath: File | DesktopFilePath) => { +const generateVideoThumbnail = async (blob: Blob) => { try { - return await FFmpegService.generateVideoThumbnail(fileOrPath); + return await ffmpeg.generateVideoThumbnail(blob); } catch (e) { log.error( - `Failed to generate thumbnail using FFmpeg for ${fopLabel(fileOrPath)}`, + `Failed to generate video thumbnail using FFmpeg, will fallback to canvas`, e, ); - // If we're on the web, try falling back to using the canvas instead. - if (fileOrPath instanceof File) { - log.info(); - } - - return await generateVideoThumbnailUsingCanvas(file); + return generateVideoThumbnailUsingCanvas(blob); } - return thumbnail; }; -async function generateVideoThumbnailUsingCanvas(file: File | ElectronFile) { +const generateVideoThumbnailUsingCanvas = async (blob: Blob) => { const canvas = document.createElement("canvas"); - const canvasCTX = canvas.getContext("2d"); + const canvasCtx = canvas.getContext("2d"); - let timeout = null; - let videoURL = null; - - let video = document.createElement("video"); - videoURL = URL.createObjectURL(new Blob([await file.arrayBuffer()])); - await new Promise((resolve, reject) => { - video.preload = "metadata"; - video.src = videoURL; - video.addEventListener("loadeddata", function () { - try { - URL.revokeObjectURL(videoURL); - if (!video) { - throw Error("video load failed"); + const videoURL = URL.createObjectURL(blob); + await withTimeout( + new Promise((resolve, reject) => { + const video = document.createElement("video"); + video.preload = "metadata"; + video.src = videoURL; + video.addEventListener("loadeddata", () => { + try { + URL.revokeObjectURL(videoURL); + const { width, height } = scaledThumbnailDimensions( + video.videoWidth, + video.videoHeight, + maxThumbnailDimension, + ); + canvas.width = width; + canvas.height = height; + canvasCtx.drawImage(video, 0, 0, width, height); + resolve(undefined); + } catch (e) { + reject(e); } - const { width, height } = scaledThumbnailDimensions( - video.videoWidth, - video.videoHeight, - maxThumbnailDimension, - ); - canvas.width = width; - canvas.height = height; - canvasCTX.drawImage(video, 0, 0, width, height); - video = null; - clearTimeout(timeout); - resolve(null); - } catch (e) { - const err = Error( - `${CustomError.THUMBNAIL_GENERATION_FAILED} err: ${e}`, - ); - log.error(CustomError.THUMBNAIL_GENERATION_FAILED, e); - reject(err); - } - }); - timeout = setTimeout( - () => reject(new Error("Operation timed out")), - WAIT_TIME_THUMBNAIL_GENERATION, - ); - }); - const thumbnailBlob = await getCompressedThumbnailBlobFromCanvas(canvas); - return await getUint8ArrayView(thumbnailBlob); -} - -const compressedJPEGData = async (canvas: HTMLCanvasElement) => { - let blob: Blob; - let prevSize = Number.MAX_SAFE_INTEGER; - let quality = 0.7; - - do { - if (blob) prevSize = blob.size; - blob = await new Promise((resolve) => { - canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality); - }); - quality -= 0.1; - } while ( - quality >= 0.5 && - blob.size > maxThumbnailSize && - percentageSizeDiff(blob.size, prevSize) >= 10 + }); + }), + 30 * 1000, ); - return blob; + return await compressedJPEGData(canvas); }; -const percentageSizeDiff = ( - newThumbnailSize: number, - oldThumbnailSize: number, -) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; - /** * Compute the size of the thumbnail to create for an image with the given * {@link width} and {@link height}. @@ -286,3 +237,28 @@ const scaledThumbnailDimensions = ( return { width: 0, height: 0 }; return thumbnailDimensions; }; + +const compressedJPEGData = async (canvas: HTMLCanvasElement) => { + let blob: Blob; + let prevSize = Number.MAX_SAFE_INTEGER; + let quality = 0.7; + + do { + if (blob) prevSize = blob.size; + blob = await new Promise((resolve) => { + canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality); + }); + quality -= 0.1; + } while ( + quality >= 0.5 && + blob.size > maxThumbnailSize && + percentageSizeDiff(blob.size, prevSize) >= 10 + ); + + return blob; +}; + +const percentageSizeDiff = ( + newThumbnailSize: number, + oldThumbnailSize: number, +) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; From 4750caf156f0698458c923bd8cfa5cbfca7ffe35 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 16:11:08 +0530 Subject: [PATCH 09/82] Blob --- desktop/src/main/utils-temp.ts | 15 ++--- web/apps/photos/src/services/ffmpeg.ts | 25 ++++--- .../photos/src/services/upload/thumbnail.ts | 17 ++--- web/apps/photos/src/worker/ffmpeg.worker.ts | 66 ++++++++----------- web/packages/shared/utils/temp.ts | 14 ---- 5 files changed, 49 insertions(+), 88 deletions(-) delete mode 100644 web/packages/shared/utils/temp.ts diff --git a/desktop/src/main/utils-temp.ts b/desktop/src/main/utils-temp.ts index 35455e85e1..e9c4c17736 100644 --- a/desktop/src/main/utils-temp.ts +++ b/desktop/src/main/utils-temp.ts @@ -13,17 +13,14 @@ const enteTempDirPath = async () => { return result; }; -const randomPrefix = (length: number) => { - const CHARACTERS = +/** Generate a random string suitable for being used as a file name prefix */ +const randomPrefix = () => { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; let result = ""; - const charactersLength = CHARACTERS.length; - for (let i = 0; i < length; i++) { - result += CHARACTERS.charAt( - Math.floor(Math.random() * charactersLength), - ); - } + for (let i = 0; i < 10; i++) + result += alphabet[Math.floor(Math.random() * alphabet.length)]; return result; }; @@ -41,7 +38,7 @@ export const makeTempFilePath = async (formatSuffix: string) => { const tempDir = await enteTempDirPath(); let result: string; do { - result = path.join(tempDir, randomPrefix(10) + "-" + formatSuffix); + result = path.join(tempDir, randomPrefix() + "-" + formatSuffix); } while (existsSync(result)); return result; }; diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 6a35f65697..5dc40b0e56 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,4 +1,4 @@ -import { ElectronFile, type DesktopFilePath } from "@/next/types/file"; +import { ElectronFile } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -17,12 +17,10 @@ import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; * This function is called during upload, when we need to generate thumbnails * for the new files that the user is adding. * - * @param fileOrPath The input video file or a path to it. - * @returns JPEG data for the generated thumbnail. + * @param blob The input video blob. + * @returns JPEG data of the generated thumbnail. */ -export const generateVideoThumbnail = async ( - fileOrPath: File | DesktopFilePath, -): Promise => { +export const generateVideoThumbnail = async (blob: Blob) => { const thumbnailAtTime = (seekTime: number) => ffmpegExec( [ @@ -37,7 +35,7 @@ export const generateVideoThumbnail = async ( "scale=-1:720", outputPathPlaceholder, ], - fileOrPath, + blob, "thumb.jpeg", ); @@ -171,20 +169,19 @@ export async function convertToMP4(file: File) { */ const ffmpegExec = async ( command: string[], - fileOrPath: File | DesktopFilePath, + blob: Blob, outputFileName: string, timeoutMs: number = 0, ): Promise => { - if (fileOrPath instanceof File) { + const electron = globalThis.electron; + if (electron) + return electron.ffmpegExec(command, blob, outputFileName, timeoutMs); + else return workerFactory .lazy() .then((worker) => - worker.exec(command, fileOrPath, outputFileName, timeoutMs), + worker.exec(command, blob, outputFileName, timeoutMs), ); - } else { - const { path, electron } = fileOrPath; - return electron.ffmpegExec(command, path, outputFileName, timeoutMs); - } }; const ffmpegExec2 = async ( diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index e10e81f536..9aad060b61 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,5 +1,4 @@ import log from "@/next/log"; -import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; import { withTimeout } from "@ente/shared/utils"; import { FILE_TYPE } from "constants/file"; @@ -84,16 +83,11 @@ const generateImageThumbnail = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ) => { - let jpegData: Uint8Array | undefined; - const electron = globalThis.electron; const available = !moduleState.isNativeThumbnailCreationNotAvailable; if (electron && available) { - // If we're running in our desktop app, try to make the thumbnail using - // the native tools available there-in, it'll be faster than doing it on - // the web layer. try { - jpegData = await generateImageThumbnailInElectron(electron, file); + return await generateImageThumbnailInElectron(electron, file); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { moduleState.isNativeThumbnailCreationNotAvailable = true; @@ -103,15 +97,12 @@ const generateImageThumbnail = async ( } } - if (!jpegData) { - jpegData = await generateImageThumbnailUsingCanvas(blob, fileTypeInfo); - } - return jpegData; + return await generateImageThumbnailUsingCanvas(blob, fileTypeInfo); }; const generateImageThumbnailInElectron = async ( electron: Electron, - inputFile: File | ElectronFile, + blob: Blob, ): Promise => { const startTime = Date.now(); const jpegData = await electron.generateImageThumbnail( @@ -255,7 +246,7 @@ const compressedJPEGData = async (canvas: HTMLCanvasElement) => { percentageSizeDiff(blob.size, prevSize) >= 10 ); - return blob; + return new Uint8Array(await blob.arrayBuffer()); }; const percentageSizeDiff = ( diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index cd608f6bc4..b30b2fa38f 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -1,8 +1,6 @@ -import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { withTimeout } from "@ente/shared/utils"; import QueueProcessor from "@ente/shared/utils/queueProcessor"; -import { generateTempName } from "@ente/shared/utils/temp"; import { expose } from "comlink"; import { ffmpegPathPlaceholder, @@ -23,21 +21,15 @@ export class DedicatedFFmpegWorker { } /** - * Execute a FFmpeg {@link command}. + * Execute a FFmpeg {@link command} on {@link blob}. * - * This is a sibling of {@link ffmpegExec} in ipc.ts exposed by the desktop - * app. See [Note: FFmpeg in Electron]. + * This is a sibling of {@link ffmpegExec} exposed by the desktop app in + * `ipc.ts`. See [Note: FFmpeg in Electron]. */ - async exec( - command: string[], - inputFile: File, - outputFileName: string, - timeoutMs, - ): Promise { + async exec(command: string[], blob: Blob, timeoutMs): Promise { if (!this.ffmpeg.isLoaded()) await this.ffmpeg.load(); - const go = () => - ffmpegExec(this.ffmpeg, command, inputFile, outputFileName); + const go = () => ffmpegExec(this.ffmpeg, command, blob); const request = this.ffmpegTaskQueue.queueUpRequest(() => timeoutMs ? withTimeout(go(), timeoutMs) : go(), @@ -49,48 +41,46 @@ export class DedicatedFFmpegWorker { expose(DedicatedFFmpegWorker, self); -const ffmpegExec = async ( - ffmpeg: FFmpeg, - command: string[], - inputFile: File, - outputFileName: string, -) => { - const [, extension] = nameAndExtension(inputFile.name); - const tempNameSuffix = extension ? `input.${extension}` : "input"; - const tempInputFilePath = `${generateTempName(10, tempNameSuffix)}`; - const tempOutputFilePath = `${generateTempName(10, outputFileName)}`; +const ffmpegExec = async (ffmpeg: FFmpeg, command: string[], blob: Blob) => { + const inputPath = `${randomPrefix()}.in`; + const outputPath = `${randomPrefix()}.out`; - const cmd = substitutePlaceholders( - command, - tempInputFilePath, - tempOutputFilePath, - ); + const cmd = substitutePlaceholders(command, inputPath, outputPath); + + const inputData = new Uint8Array(await blob.arrayBuffer()); try { - ffmpeg.FS( - "writeFile", - tempInputFilePath, - new Uint8Array(await inputFile.arrayBuffer()), - ); + ffmpeg.FS("writeFile", inputPath, inputData); log.info(`Running FFmpeg (wasm) command ${cmd}`); await ffmpeg.run(...cmd); - return ffmpeg.FS("readFile", tempOutputFilePath); + return ffmpeg.FS("readFile", outputPath); } finally { try { - ffmpeg.FS("unlink", tempInputFilePath); + ffmpeg.FS("unlink", inputPath); } catch (e) { - log.error("Failed to remove input ${tempInputFilePath}", e); + log.error(`Failed to remove input ${inputPath}`, e); } try { - ffmpeg.FS("unlink", tempOutputFilePath); + ffmpeg.FS("unlink", outputPath); } catch (e) { - log.error("Failed to remove output ${tempOutputFilePath}", e); + log.error(`Failed to remove output ${outputPath}`, e); } } }; +/** Generate a random string suitable for being used as a file name prefix */ +const randomPrefix = () => { + const alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + let result = ""; + for (let i = 0; i < 10; i++) + result += alphabet[Math.floor(Math.random() * alphabet.length)]; + return result; +}; + const substitutePlaceholders = ( command: string[], inputFilePath: string, diff --git a/web/packages/shared/utils/temp.ts b/web/packages/shared/utils/temp.ts deleted file mode 100644 index 984f4abb05..0000000000 --- a/web/packages/shared/utils/temp.ts +++ /dev/null @@ -1,14 +0,0 @@ -const CHARACTERS = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - -export function generateTempName(length: number, suffix: string) { - let tempName = ""; - - const charactersLength = CHARACTERS.length; - for (let i = 0; i < length; i++) { - tempName += CHARACTERS.charAt( - Math.floor(Math.random() * charactersLength), - ); - } - return `${tempName}-${suffix}`; -} From 4461775283921f5cd2e82206884ed662df4ed35d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 16:32:04 +0530 Subject: [PATCH 10/82] Desktop side --- desktop/src/main/ipc.ts | 5 ++--- desktop/src/main/services/convert.ts | 4 ++-- desktop/src/main/services/ffmpeg.ts | 13 ++++++------- desktop/src/main/services/ml-clip.ts | 2 +- desktop/src/main/utils-temp.ts | 13 +++++++------ desktop/src/preload.ts | 11 ++--------- web/apps/photos/src/services/ffmpeg.ts | 27 ++++++++++++-------------- web/packages/next/types/ipc.ts | 25 ++++++++++-------------- 8 files changed, 42 insertions(+), 58 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 9ea4d802fa..ed6ea48cb2 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -156,10 +156,9 @@ export const attachIPCHandlers = () => { ( _, command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, timeoutMS: number, - ) => ffmpegExec(command, inputDataOrPath, outputFileName, timeoutMS), + ) => ffmpegExec(command, dataOrPath, timeoutMS), ); // - ML diff --git a/desktop/src/main/services/convert.ts b/desktop/src/main/services/convert.ts index 7f38a86ea6..71e8f419e1 100644 --- a/desktop/src/main/services/convert.ts +++ b/desktop/src/main/services/convert.ts @@ -14,7 +14,7 @@ export const convertToJPEG = async ( imageData: Uint8Array, ): Promise => { const inputFilePath = await makeTempFilePath(fileName); - const outputFilePath = await makeTempFilePath("output.jpeg"); + const outputFilePath = await makeTempFilePath(".jpeg"); // Construct the command first, it may throw on NotAvailable on win32. const command = convertToJPEGCommand(inputFilePath, outputFilePath); @@ -150,7 +150,7 @@ async function generateImageThumbnail_( let tempOutputFilePath: string; let quality = MAX_QUALITY; try { - tempOutputFilePath = await makeTempFilePath("thumb.jpeg"); + tempOutputFilePath = await makeTempFilePath(".jpeg"); let thumbnail: Uint8Array; do { await execAsync( diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index 5547cf8341..251f71f6d2 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -37,8 +37,7 @@ const outputPathPlaceholder = "OUTPUT"; */ export const ffmpegExec = async ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, timeoutMS: number, ): Promise => { // TODO (MR): This currently copies files for both input and output. This @@ -47,18 +46,18 @@ export const ffmpegExec = async ( let inputFilePath: string; let isInputFileTemporary: boolean; - if (typeof inputDataOrPath == "string") { - inputFilePath = inputDataOrPath; + if (typeof dataOrPath == "string") { + inputFilePath = dataOrPath; isInputFileTemporary = false; } else { - inputFilePath = await makeTempFilePath("input" /* arbitrary */); + inputFilePath = await makeTempFilePath(".in"); isInputFileTemporary = true; - await fs.writeFile(inputFilePath, inputDataOrPath); + await fs.writeFile(inputFilePath, dataOrPath); } let outputFilePath: string | undefined; try { - outputFilePath = await makeTempFilePath(outputFileName); + outputFilePath = await makeTempFilePath(".out"); const cmd = substitutePlaceholders( command, diff --git a/desktop/src/main/services/ml-clip.ts b/desktop/src/main/services/ml-clip.ts index 0c466b9f66..a5f407f9e7 100644 --- a/desktop/src/main/services/ml-clip.ts +++ b/desktop/src/main/services/ml-clip.ts @@ -20,7 +20,7 @@ const cachedCLIPImageSession = makeCachedInferenceSession( ); export const clipImageEmbedding = async (jpegImageData: Uint8Array) => { - const tempFilePath = await makeTempFilePath(""); + const tempFilePath = await makeTempFilePath(); const imageStream = new Response(jpegImageData.buffer).body; await writeStream(tempFilePath, imageStream); try { diff --git a/desktop/src/main/utils-temp.ts b/desktop/src/main/utils-temp.ts index e9c4c17736..f48b2c388f 100644 --- a/desktop/src/main/utils-temp.ts +++ b/desktop/src/main/utils-temp.ts @@ -25,20 +25,21 @@ const randomPrefix = () => { }; /** - * Return the path to a temporary file with the given {@link formatSuffix}. + * Return the path to a temporary file with the given {@link suffix}. * * The function returns the path to a file in the system temp directory (in an - * Ente specific folder therin) with a random prefix and the given - * {@link formatSuffix}. It ensures that there is no existing file with the same - * name already. + * Ente specific folder therin) with a random prefix and an (optional) + * {@link suffix}. + * + * It ensures that there is no existing file with the same name already. * * Use {@link deleteTempFile} to remove this file when you're done. */ -export const makeTempFilePath = async (formatSuffix: string) => { +export const makeTempFilePath = async (suffix?: string) => { const tempDir = await enteTempDirPath(); let result: string; do { - result = path.join(tempDir, randomPrefix() + "-" + formatSuffix); + result = path.join(tempDir, `${randomPrefix()}${suffix ?? ""}`); } while (existsSync(result)); return result; }; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index c3f964e17d..c2af31abcc 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -144,17 +144,10 @@ const generateImageThumbnail = ( const ffmpegExec = ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, timeoutMS: number, ): Promise => - ipcRenderer.invoke( - "ffmpegExec", - command, - inputDataOrPath, - outputFileName, - timeoutMS, - ); + ipcRenderer.invoke("ffmpegExec", command, dataOrPath, timeoutMS); // - ML diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 5dc40b0e56..3c3f58a5fb 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -36,7 +36,6 @@ export const generateVideoThumbnail = async (blob: Blob) => { outputPathPlaceholder, ], blob, - "thumb.jpeg", ); try { @@ -161,27 +160,25 @@ export async function convertToMP4(file: File) { * Run the given FFmpeg command. * * If we're running in the context of our desktop app, use the FFmpeg binary we - * bundle with our desktop app to run the command. Otherwise fallback to using - * the wasm FFmpeg we link to from our web app in a web worker. + * bundle with our desktop app to run the command. Otherwise fallback to using a + * wasm FFmpeg in a web worker. * - * As a rough ballpark, the native FFmpeg integration in the desktop app is - * 10-20x faster than the wasm one currently. See: [Note: FFmpeg in Electron]. + * As a rough ballpark, currently the native FFmpeg integration in the desktop + * app is 10-20x faster than the wasm one. See: [Note: FFmpeg in Electron]. */ const ffmpegExec = async ( command: string[], blob: Blob, - outputFileName: string, timeoutMs: number = 0, -): Promise => { +) => { const electron = globalThis.electron; - if (electron) - return electron.ffmpegExec(command, blob, outputFileName, timeoutMs); - else - return workerFactory - .lazy() - .then((worker) => - worker.exec(command, blob, outputFileName, timeoutMs), - ); + if (electron) { + const data = new Uint8Array(await blob.arrayBuffer()); + return await electron.ffmpegExec(command, data, timeoutMs); + } else { + const worker = await workerFactory.lazy() + return await worker.exec(command, blob, timeoutMs); + } }; const ffmpegExec2 = async ( diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index d87b8e830d..ef10a43fe8 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -237,11 +237,11 @@ export interface Electron { ) => Promise; /** - * Execute a ffmpeg {@link command}. + * Execute a FFmpeg {@link command} on the given {@link dataOrPath}. * - * This executes the command using the ffmpeg executable we bundle with our - * desktop app. There is also a ffmpeg wasm implementation that we use when - * running on the web, it also has a sibling function with the same + * This executes the command using a FFmpeg executable we bundle with our + * desktop app. We also have a wasm FFmpeg wasm implementation that we use + * when running on the web, which has a sibling function with the same * parameters. See [Note: ffmpeg in Electron]. * * @param command An array of strings, each representing one positional @@ -250,25 +250,20 @@ export interface Electron { * (respectively {@link inputPathPlaceholder}, * {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}). * - * @param inputDataOrPath The bytes of the input file, or the path to the - * input file on the user's local disk. In both cases, the data gets - * serialized to a temporary file, and then that path gets substituted in - * the ffmpeg {@link command} by {@link inputPathPlaceholder}. - * - * @param outputFileName The name of the file we instruct ffmpeg to produce - * when giving it the given {@link command}. The contents of this file get - * returned as the result. + * @param dataOrPath The bytes of the input file, or the path to the input + * file on the user's local disk. In both cases, the data gets serialized to + * a temporary file, and then that path gets substituted in the FFmpeg + * {@link command} in lieu of {@link inputPathPlaceholder}. * * @param timeoutMS If non-zero, then abort and throw a timeout error if the * ffmpeg command takes more than the given number of milliseconds. * * @returns The contents of the output file produced by the ffmpeg command - * at {@link outputFileName}. + * (specified as {@link outputPathPlaceholder} in {@link command}). */ ffmpegExec: ( command: string[], - inputDataOrPath: Uint8Array | string, - outputFileName: string, + dataOrPath: Uint8Array | string, timeoutMS: number, ) => Promise; From 05cd0bcd2cdff20dd58d49ed0534750df051b6fc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 16:49:01 +0530 Subject: [PATCH 11/82] input filename is not needed tested with sips --- desktop/src/main/ipc.ts | 12 ++++--- desktop/src/main/services/convert.ts | 31 ++++++++++++++----- desktop/src/preload.ts | 11 +++---- .../photos/src/services/upload/thumbnail.ts | 7 ++--- web/apps/photos/src/utils/file/index.ts | 16 +++++----- web/packages/next/types/ipc.ts | 12 +++---- web/packages/next/worker/comlink-worker.ts | 4 +-- 7 files changed, 51 insertions(+), 42 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index ed6ea48cb2..e55a26f154 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -141,14 +141,18 @@ export const attachIPCHandlers = () => { // - Conversion - ipcMain.handle("convertToJPEG", (_, fileName, imageData) => - convertToJPEG(fileName, imageData), + ipcMain.handle("convertToJPEG", (_, imageData: Uint8Array) => + convertToJPEG(imageData), ); ipcMain.handle( "generateImageThumbnail", - (_, inputFile, maxDimension, maxSize) => - generateImageThumbnail(inputFile, maxDimension, maxSize), + ( + _, + dataOrPath: Uint8Array | string, + maxDimension: number, + maxSize: number, + ) => generateImageThumbnail(dataOrPath, maxDimension, maxSize), ); ipcMain.handle( diff --git a/desktop/src/main/services/convert.ts b/desktop/src/main/services/convert.ts index 71e8f419e1..689d73fb09 100644 --- a/desktop/src/main/services/convert.ts +++ b/desktop/src/main/services/convert.ts @@ -3,20 +3,17 @@ import { existsSync } from "fs"; import fs from "node:fs/promises"; import path from "path"; -import { CustomErrorMessage, ElectronFile } from "../../types/ipc"; +import { CustomErrorMessage } from "../../types/ipc"; import log from "../log"; import { writeStream } from "../stream"; import { execAsync, isDev } from "../utils-electron"; import { deleteTempFile, makeTempFilePath } from "../utils-temp"; -export const convertToJPEG = async ( - fileName: string, - imageData: Uint8Array, -): Promise => { - const inputFilePath = await makeTempFilePath(fileName); +export const convertToJPEG = async (imageData: Uint8Array) => { + const inputFilePath = await makeTempFilePath(); const outputFilePath = await makeTempFilePath(".jpeg"); - // Construct the command first, it may throw on NotAvailable on win32. + // Construct the command first, it may throw NotAvailable on win32. const command = convertToJPEGCommand(inputFilePath, outputFilePath); try { @@ -106,10 +103,28 @@ const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ ]; export async function generateImageThumbnail( - inputFile: File | ElectronFile, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ): Promise { + const inputFilePath = await makeTempFilePath(fileName); + const outputFilePath = await makeTempFilePath(".jpeg"); + + // Construct the command first, it may throw NotAvailable on win32. + const command = convertToJPEGCommand(inputFilePath, outputFilePath); + + try { + await fs.writeFile(inputFilePath, imageData); + await execAsync(command); + return new Uint8Array(await fs.readFile(outputFilePath)); + } finally { + try { + deleteTempFile(outputFilePath); + deleteTempFile(inputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } + } let inputFilePath = null; let createdTempInputFile = null; try { diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index c2af31abcc..728e8d012b 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -124,20 +124,17 @@ const fsIsDir = (dirPath: string): Promise => // - Conversion -const convertToJPEG = ( - fileName: string, - imageData: Uint8Array, -): Promise => - ipcRenderer.invoke("convertToJPEG", fileName, imageData); +const convertToJPEG = (imageData: Uint8Array): Promise => + ipcRenderer.invoke("convertToJPEG", imageData); const generateImageThumbnail = ( - inputFile: File | ElectronFile, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ): Promise => ipcRenderer.invoke( "generateImageThumbnail", - inputFile, + dataOrPath, maxDimension, maxSize, ); diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 9aad060b61..a23e68a2e3 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -59,15 +59,12 @@ export const generateThumbnail = async ( const thumbnail = fileTypeInfo.fileType === FILE_TYPE.IMAGE ? await generateImageThumbnail(blob, fileTypeInfo) - : await generateVideoThumbnail(blob, fileTypeInfo); + : await generateVideoThumbnail(blob); if (thumbnail.length == 0) throw new Error("Empty thumbnail"); return { thumbnail, hasStaticThumbnail: false }; } catch (e) { - log.error( - `Failed to generate thumbnail for format ${fileTypeInfo.exactType}`, - e, - ); + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; } }; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 5c715fa482..a6cb640b6a 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -292,14 +292,12 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { return imageBlob; } - let jpegBlob: Blob | undefined; - const available = !moduleState.isNativeJPEGConversionNotAvailable; if (isElectron() && available && isSupportedRawFormat(exactType)) { // If we're running in our desktop app, see if our Node.js layer can // convert this into a JPEG using native tools for us. try { - jpegBlob = await nativeConvertToJPEG(fileName, imageBlob); + return await nativeConvertToJPEG(imageBlob); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { moduleState.isNativeJPEGConversionNotAvailable = true; @@ -309,12 +307,12 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } } - if (!jpegBlob && isFileHEIC(exactType)) { + if (!isFileHEIC(exactType)) { // If it is an HEIC file, use our web HEIC converter. - jpegBlob = await heicToJPEG(imageBlob); + return await heicToJPEG(imageBlob); } - return jpegBlob; + return undefined; } catch (e) { log.error( `Failed to get renderable image for ${JSON.stringify(fileTypeInfo ?? fileName)}`, @@ -324,7 +322,7 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } }; -const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => { +const nativeConvertToJPEG = async (imageBlob: Blob) => { const startTime = Date.now(); const imageData = new Uint8Array(await imageBlob.arrayBuffer()); const electron = globalThis.electron; @@ -332,8 +330,8 @@ const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => { // the main thread since workers don't have access to the `window` (and // thus, to the `window.electron`) object. const jpegData = electron - ? await electron.convertToJPEG(fileName, imageData) - : await workerBridge.convertToJPEG(fileName, imageData); + ? await electron.convertToJPEG(imageData) + : await workerBridge.convertToJPEG(imageData); log.debug(() => `Native JPEG conversion took ${Date.now() - startTime} ms`); return new Blob([jpegData]); }; diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index ef10a43fe8..b7ad3c0f5e 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -204,14 +204,10 @@ export interface Electron { * yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param fileName The name of the file whose data we're being given. * @param imageData The raw image data (the contents of the image file). * @returns JPEG data of the converted image. */ - convertToJPEG: ( - fileName: string, - imageData: Uint8Array, - ) => Promise; + convertToJPEG: (imageData: Uint8Array) => Promise; /** * Generate a JPEG thumbnail for the given image. @@ -224,14 +220,16 @@ export interface Electron { * not yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param inputFile The file whose thumbnail we want. + * @param dataOrPath The data-of or path-to the image whose thumbnail we + * want. * @param maxDimension The maximum width or height of the generated * thumbnail. * @param maxSize Maximum size (in bytes) of the generated thumbnail. + * * @returns JPEG data of the generated thumbnail. */ generateImageThumbnail: ( - inputFile: File | ElectronFile, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ) => Promise; diff --git a/web/packages/next/worker/comlink-worker.ts b/web/packages/next/worker/comlink-worker.ts index 7bae126a40..5929e5361b 100644 --- a/web/packages/next/worker/comlink-worker.ts +++ b/web/packages/next/worker/comlink-worker.ts @@ -44,8 +44,8 @@ const workerBridge = { logToDisk, // Needed by ML worker getAuthToken: () => ensureLocalUser().then((user) => user.token), - convertToJPEG: (fileName: string, imageData: Uint8Array) => - ensureElectron().convertToJPEG(fileName, imageData), + convertToJPEG: (imageData: Uint8Array) => + ensureElectron().convertToJPEG(imageData), detectFaces: (input: Float32Array) => ensureElectron().detectFaces(input), faceEmbedding: (input: Float32Array) => ensureElectron().faceEmbedding(input), From dfa50e8ed1658034b5457a408288eeea02c3b2f8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 17:14:21 +0530 Subject: [PATCH 12/82] thumb --- desktop/src/main/ipc.ts | 2 +- desktop/src/main/services/convert.ts | 256 --------------------------- desktop/src/main/services/ffmpeg.ts | 16 +- desktop/src/main/services/image.ts | 160 +++++++++++++++++ 4 files changed, 171 insertions(+), 263 deletions(-) delete mode 100644 desktop/src/main/services/convert.ts create mode 100644 desktop/src/main/services/image.ts diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index e55a26f154..946f965781 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -38,7 +38,7 @@ import { updateAndRestart, updateOnNextRestart, } from "./services/app-update"; -import { convertToJPEG, generateImageThumbnail } from "./services/convert"; +import { convertToJPEG, generateImageThumbnail } from "./services/image"; import { ffmpegExec } from "./services/ffmpeg"; import { getDirFiles } from "./services/fs"; import { diff --git a/desktop/src/main/services/convert.ts b/desktop/src/main/services/convert.ts deleted file mode 100644 index 689d73fb09..0000000000 --- a/desktop/src/main/services/convert.ts +++ /dev/null @@ -1,256 +0,0 @@ -/** @file Image conversions */ - -import { existsSync } from "fs"; -import fs from "node:fs/promises"; -import path from "path"; -import { CustomErrorMessage } from "../../types/ipc"; -import log from "../log"; -import { writeStream } from "../stream"; -import { execAsync, isDev } from "../utils-electron"; -import { deleteTempFile, makeTempFilePath } from "../utils-temp"; - -export const convertToJPEG = async (imageData: Uint8Array) => { - const inputFilePath = await makeTempFilePath(); - const outputFilePath = await makeTempFilePath(".jpeg"); - - // Construct the command first, it may throw NotAvailable on win32. - const command = convertToJPEGCommand(inputFilePath, outputFilePath); - - try { - await fs.writeFile(inputFilePath, imageData); - await execAsync(command); - return new Uint8Array(await fs.readFile(outputFilePath)); - } finally { - try { - deleteTempFile(outputFilePath); - deleteTempFile(inputFilePath); - } catch (e) { - log.error("Ignoring error when cleaning up temp files", e); - } - } -}; - -const convertToJPEGCommand = ( - inputFilePath: string, - outputFilePath: string, -) => { - switch (process.platform) { - case "darwin": - return [ - "sips", - "-s", - "format", - "jpeg", - inputFilePath, - "--out", - outputFilePath, - ]; - case "linux": - return [ - imageMagickPath(), - inputFilePath, - "-quality", - "100%", - outputFilePath, - ]; - default: // "win32" - throw new Error(CustomErrorMessage.NotAvailable); - } -}; - -/** Path to the Linux image-magick executable bundled with our app */ -const imageMagickPath = () => - path.join(isDev ? "build" : process.resourcesPath, "image-magick"); - -const IMAGE_MAGICK_PLACEHOLDER = "IMAGE_MAGICK"; -const MAX_DIMENSION_PLACEHOLDER = "MAX_DIMENSION"; -const SAMPLE_SIZE_PLACEHOLDER = "SAMPLE_SIZE"; -const INPUT_PATH_PLACEHOLDER = "INPUT"; -const OUTPUT_PATH_PLACEHOLDER = "OUTPUT"; -const QUALITY_PLACEHOLDER = "QUALITY"; - -const MAX_QUALITY = 70; -const MIN_QUALITY = 50; - -const SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ - "sips", - "-s", - "format", - "jpeg", - "-s", - "formatOptions", - QUALITY_PLACEHOLDER, - "-Z", - MAX_DIMENSION_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - "--out", - OUTPUT_PATH_PLACEHOLDER, -]; - -const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ - IMAGE_MAGICK_PLACEHOLDER, - INPUT_PATH_PLACEHOLDER, - "-auto-orient", - "-define", - `jpeg:size=${SAMPLE_SIZE_PLACEHOLDER}x${SAMPLE_SIZE_PLACEHOLDER}`, - "-thumbnail", - `${MAX_DIMENSION_PLACEHOLDER}x${MAX_DIMENSION_PLACEHOLDER}>`, - "-unsharp", - "0x.5", - "-quality", - QUALITY_PLACEHOLDER, - OUTPUT_PATH_PLACEHOLDER, -]; - -export async function generateImageThumbnail( - dataOrPath: Uint8Array | string, - maxDimension: number, - maxSize: number, -): Promise { - const inputFilePath = await makeTempFilePath(fileName); - const outputFilePath = await makeTempFilePath(".jpeg"); - - // Construct the command first, it may throw NotAvailable on win32. - const command = convertToJPEGCommand(inputFilePath, outputFilePath); - - try { - await fs.writeFile(inputFilePath, imageData); - await execAsync(command); - return new Uint8Array(await fs.readFile(outputFilePath)); - } finally { - try { - deleteTempFile(outputFilePath); - deleteTempFile(inputFilePath); - } catch (e) { - log.error("Ignoring error when cleaning up temp files", e); - } - } - let inputFilePath = null; - let createdTempInputFile = null; - try { - if (process.platform == "win32") - throw Error( - CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED, - ); - if (!existsSync(inputFile.path)) { - const tempFilePath = await makeTempFilePath(inputFile.name); - await writeStream(tempFilePath, await inputFile.stream()); - inputFilePath = tempFilePath; - createdTempInputFile = true; - } else { - inputFilePath = inputFile.path; - } - const thumbnail = await generateImageThumbnail_( - inputFilePath, - maxDimension, - maxSize, - ); - return thumbnail; - } finally { - if (createdTempInputFile) { - try { - await deleteTempFile(inputFilePath); - } catch (e) { - log.error(`Failed to deleteTempFile ${inputFilePath}`, e); - } - } - } -} - -async function generateImageThumbnail_( - inputFilePath: string, - width: number, - maxSize: number, -): Promise { - let tempOutputFilePath: string; - let quality = MAX_QUALITY; - try { - tempOutputFilePath = await makeTempFilePath(".jpeg"); - let thumbnail: Uint8Array; - do { - await execAsync( - constructThumbnailGenerationCommand( - inputFilePath, - tempOutputFilePath, - width, - quality, - ), - ); - thumbnail = new Uint8Array(await fs.readFile(tempOutputFilePath)); - quality -= 10; - } while (thumbnail.length > maxSize && quality > MIN_QUALITY); - return thumbnail; - } catch (e) { - log.error("Failed to generate image thumbnail", e); - throw e; - } finally { - try { - await fs.rm(tempOutputFilePath, { force: true }); - } catch (e) { - log.error( - `Failed to remove tempOutputFile ${tempOutputFilePath}`, - e, - ); - } - } -} - -function constructThumbnailGenerationCommand( - inputFilePath: string, - tempOutputFilePath: string, - maxDimension: number, - quality: number, -) { - let thumbnailGenerationCmd: string[]; - if (process.platform == "darwin") { - thumbnailGenerationCmd = SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map( - (cmdPart) => { - if (cmdPart === INPUT_PATH_PLACEHOLDER) { - return inputFilePath; - } - if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { - return tempOutputFilePath; - } - if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { - return maxDimension.toString(); - } - if (cmdPart === QUALITY_PLACEHOLDER) { - return quality.toString(); - } - return cmdPart; - }, - ); - } else if (process.platform == "linux") { - thumbnailGenerationCmd = - IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map((cmdPart) => { - if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { - return imageMagickPath(); - } - if (cmdPart === INPUT_PATH_PLACEHOLDER) { - return inputFilePath; - } - if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { - return tempOutputFilePath; - } - if (cmdPart.includes(SAMPLE_SIZE_PLACEHOLDER)) { - return cmdPart.replaceAll( - SAMPLE_SIZE_PLACEHOLDER, - (2 * maxDimension).toString(), - ); - } - if (cmdPart.includes(MAX_DIMENSION_PLACEHOLDER)) { - return cmdPart.replaceAll( - MAX_DIMENSION_PLACEHOLDER, - maxDimension.toString(), - ); - } - if (cmdPart === QUALITY_PLACEHOLDER) { - return quality.toString(); - } - return cmdPart; - }); - } else { - throw new Error(`Unsupported OS ${process.platform}`); - } - return thumbnailGenerationCmd; -} diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index 251f71f6d2..b1ae9e301a 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -1,9 +1,11 @@ import pathToFfmpeg from "ffmpeg-static"; import fs from "node:fs/promises"; +import log from "../log"; import { withTimeout } from "../utils"; import { execAsync } from "../utils-electron"; import { deleteTempFile, makeTempFilePath } from "../utils-temp"; +/* Duplicated in the web app's code (used by the WASM FFmpeg implementation). */ const ffmpegPathPlaceholder = "FFMPEG"; const inputPathPlaceholder = "INPUT"; const outputPathPlaceholder = "OUTPUT"; @@ -50,15 +52,13 @@ export const ffmpegExec = async ( inputFilePath = dataOrPath; isInputFileTemporary = false; } else { - inputFilePath = await makeTempFilePath(".in"); + inputFilePath = await makeTempFilePath(); isInputFileTemporary = true; await fs.writeFile(inputFilePath, dataOrPath); } - let outputFilePath: string | undefined; + const outputFilePath = await makeTempFilePath(); try { - outputFilePath = await makeTempFilePath(".out"); - const cmd = substitutePlaceholders( command, inputFilePath, @@ -70,8 +70,12 @@ export const ffmpegExec = async ( return fs.readFile(outputFilePath); } finally { - if (isInputFileTemporary) await deleteTempFile(inputFilePath); - if (outputFilePath) await deleteTempFile(outputFilePath); + try { + if (isInputFileTemporary) await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } } }; diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts new file mode 100644 index 0000000000..abeca5432d --- /dev/null +++ b/desktop/src/main/services/image.ts @@ -0,0 +1,160 @@ +/** @file Image format conversions and thumbnail generation */ + +import fs from "node:fs/promises"; +import path from "path"; +import { CustomErrorMessage } from "../../types/ipc"; +import log from "../log"; +import { execAsync, isDev } from "../utils-electron"; +import { deleteTempFile, makeTempFilePath } from "../utils-temp"; + +export const convertToJPEG = async (imageData: Uint8Array) => { + const inputFilePath = await makeTempFilePath(); + const outputFilePath = await makeTempFilePath(".jpeg"); + + // Construct the command first, it may throw NotAvailable on win32. + const command = convertToJPEGCommand(inputFilePath, outputFilePath); + + try { + await fs.writeFile(inputFilePath, imageData); + await execAsync(command); + return new Uint8Array(await fs.readFile(outputFilePath)); + } finally { + try { + deleteTempFile(inputFilePath); + deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } + } +}; + +const convertToJPEGCommand = ( + inputFilePath: string, + outputFilePath: string, +) => { + switch (process.platform) { + case "darwin": + return [ + "sips", + "-s", + "format", + "jpeg", + inputFilePath, + "--out", + outputFilePath, + ]; + + case "linux": + return [ + imageMagickPath(), + inputFilePath, + "-quality", + "100%", + outputFilePath, + ]; + + default: // "win32" + throw new Error(CustomErrorMessage.NotAvailable); + } +}; + +/** Path to the Linux image-magick executable bundled with our app */ +const imageMagickPath = () => + path.join(isDev ? "build" : process.resourcesPath, "image-magick"); + +export const generateImageThumbnail = async ( + dataOrPath: Uint8Array | string, + maxDimension: number, + maxSize: number, +): Promise => { + let inputFilePath: string; + let isInputFileTemporary: boolean; + if (typeof dataOrPath == "string") { + inputFilePath = dataOrPath; + isInputFileTemporary = false; + } else { + inputFilePath = await makeTempFilePath(); + isInputFileTemporary = true; + } + + const outputFilePath = await makeTempFilePath(".jpeg"); + + // Construct the command first, it may throw NotAvailable on win32. + let quality = 70; + let command = generateImageThumbnailCommand( + inputFilePath, + outputFilePath, + maxDimension, + quality, + ); + + try { + if (dataOrPath instanceof Uint8Array) + await fs.writeFile(inputFilePath, dataOrPath); + + let thumbnail: Uint8Array; + do { + await execAsync(command); + thumbnail = new Uint8Array(await fs.readFile(outputFilePath)); + quality -= 10; + command = generateImageThumbnailCommand( + inputFilePath, + outputFilePath, + maxDimension, + quality, + ); + } while (thumbnail.length > maxSize && quality > 50); + return thumbnail; + } finally { + try { + if (isInputFileTemporary) await deleteTempFile(inputFilePath); + deleteTempFile(outputFilePath); + } catch (e) { + log.error("Ignoring error when cleaning up temp files", e); + } + } +}; + +const generateImageThumbnailCommand = ( + inputFilePath: string, + outputFilePath: string, + maxDimension: number, + quality: number, +) => { + switch (process.platform) { + case "darwin": + return [ + "sips", + "-s", + "format", + "jpeg", + "-s", + "formatOptions", + `${quality}`, + "-Z", + `${maxDimension}`, + inputFilePath, + "--out", + outputFilePath, + ]; + + case "linux": + return [ + imageMagickPath(), + inputFilePath, + "-auto-orient", + "-define", + `jpeg:size=${2 * maxDimension}x${2 * maxDimension}`, + "-thumbnail", + `${maxDimension}x${maxDimension}>`, + "-unsharp", + "0x.5", + "-quality", + `${quality}`, + outputFilePath, + ]; + + default: // "win32" + throw new Error(CustomErrorMessage.NotAvailable); + } +}; From 3ab14d59498c67aabb3c5e60c66cfb9be80f70a7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 22 Apr 2024 17:19:20 +0530 Subject: [PATCH 13/82] Remove unnecessary flexibility --- desktop/src/main/ipc.ts | 10 +++----- desktop/src/main/services/image.ts | 25 ++++++------------- desktop/src/preload.ts | 4 +-- .../photos/src/services/upload/thumbnail.ts | 5 ++-- web/packages/next/types/ipc.ts | 7 +++--- 5 files changed, 20 insertions(+), 31 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 946f965781..9229d3b388 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -38,9 +38,9 @@ import { updateAndRestart, updateOnNextRestart, } from "./services/app-update"; -import { convertToJPEG, generateImageThumbnail } from "./services/image"; import { ffmpegExec } from "./services/ffmpeg"; import { getDirFiles } from "./services/fs"; +import { convertToJPEG, generateImageThumbnail } from "./services/image"; import { clipImageEmbedding, clipTextEmbeddingIfAvailable, @@ -147,12 +147,8 @@ export const attachIPCHandlers = () => { ipcMain.handle( "generateImageThumbnail", - ( - _, - dataOrPath: Uint8Array | string, - maxDimension: number, - maxSize: number, - ) => generateImageThumbnail(dataOrPath, maxDimension, maxSize), + (_, imageData: Uint8Array, maxDimension: number, maxSize: number) => + generateImageThumbnail(imageData, maxDimension, maxSize), ); ipcMain.handle( diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts index abeca5432d..47a1892449 100644 --- a/desktop/src/main/services/image.ts +++ b/desktop/src/main/services/image.ts @@ -20,8 +20,8 @@ export const convertToJPEG = async (imageData: Uint8Array) => { return new Uint8Array(await fs.readFile(outputFilePath)); } finally { try { - deleteTempFile(inputFilePath); - deleteTempFile(outputFilePath); + await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); } catch (e) { log.error("Ignoring error when cleaning up temp files", e); } @@ -63,20 +63,11 @@ const imageMagickPath = () => path.join(isDev ? "build" : process.resourcesPath, "image-magick"); export const generateImageThumbnail = async ( - dataOrPath: Uint8Array | string, + imageData: Uint8Array, maxDimension: number, maxSize: number, ): Promise => { - let inputFilePath: string; - let isInputFileTemporary: boolean; - if (typeof dataOrPath == "string") { - inputFilePath = dataOrPath; - isInputFileTemporary = false; - } else { - inputFilePath = await makeTempFilePath(); - isInputFileTemporary = true; - } - + const inputFilePath = await makeTempFilePath(); const outputFilePath = await makeTempFilePath(".jpeg"); // Construct the command first, it may throw NotAvailable on win32. @@ -89,8 +80,8 @@ export const generateImageThumbnail = async ( ); try { - if (dataOrPath instanceof Uint8Array) - await fs.writeFile(inputFilePath, dataOrPath); + if (imageData instanceof Uint8Array) + await fs.writeFile(inputFilePath, imageData); let thumbnail: Uint8Array; do { @@ -107,8 +98,8 @@ export const generateImageThumbnail = async ( return thumbnail; } finally { try { - if (isInputFileTemporary) await deleteTempFile(inputFilePath); - deleteTempFile(outputFilePath); + await deleteTempFile(inputFilePath); + await deleteTempFile(outputFilePath); } catch (e) { log.error("Ignoring error when cleaning up temp files", e); } diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 728e8d012b..0464022b00 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -128,13 +128,13 @@ const convertToJPEG = (imageData: Uint8Array): Promise => ipcRenderer.invoke("convertToJPEG", imageData); const generateImageThumbnail = ( - dataOrPath: Uint8Array | string, + imageData: Uint8Array, maxDimension: number, maxSize: number, ): Promise => ipcRenderer.invoke( "generateImageThumbnail", - dataOrPath, + imageData, maxDimension, maxSize, ); diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index a23e68a2e3..6be2ea6bdf 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -84,7 +84,7 @@ const generateImageThumbnail = async ( const available = !moduleState.isNativeThumbnailCreationNotAvailable; if (electron && available) { try { - return await generateImageThumbnailInElectron(electron, file); + return await generateImageThumbnailInElectron(electron, blob); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { moduleState.isNativeThumbnailCreationNotAvailable = true; @@ -102,8 +102,9 @@ const generateImageThumbnailInElectron = async ( blob: Blob, ): Promise => { const startTime = Date.now(); + const data = new Uint8Array(await blob.arrayBuffer()); const jpegData = await electron.generateImageThumbnail( - inputFile, + data, maxThumbnailDimension, maxThumbnailSize, ); diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index b7ad3c0f5e..6ba5c78328 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -205,6 +205,7 @@ export interface Electron { * {@link CustomErrorMessage.NotAvailable} message. * * @param imageData The raw image data (the contents of the image file). + * * @returns JPEG data of the converted image. */ convertToJPEG: (imageData: Uint8Array) => Promise; @@ -220,8 +221,8 @@ export interface Electron { * not yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param dataOrPath The data-of or path-to the image whose thumbnail we - * want. + * @param imageData The raw image data (the contents of the image file) + * whose thumbnail we want to generate. * @param maxDimension The maximum width or height of the generated * thumbnail. * @param maxSize Maximum size (in bytes) of the generated thumbnail. @@ -229,7 +230,7 @@ export interface Electron { * @returns JPEG data of the generated thumbnail. */ generateImageThumbnail: ( - dataOrPath: Uint8Array | string, + imageData: Uint8Array, maxDimension: number, maxSize: number, ) => Promise; From 7a0abf22680b4e7d8ceb0e9de9c93494c253b427 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 09:52:13 +0530 Subject: [PATCH 14/82] Prepare a split --- .../src/services/upload/uploadService.ts | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 7a118d3030..f2be02ec0f 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -144,15 +144,6 @@ class UploadService { : getFileType(file); } - async readAsset( - fileTypeInfo: FileTypeInfo, - { isLivePhoto, file, livePhotoAssets }: UploadAsset, - ) { - return isLivePhoto - ? await readLivePhoto(fileTypeInfo, livePhotoAssets) - : await readFile(fileTypeInfo, file); - } - async extractAssetMetadata( worker: Remote, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, @@ -176,12 +167,6 @@ class UploadService { ); } - constructPublicMagicMetadata( - publicMagicMetadataProps: FilePublicMagicMetadataProps, - ) { - return constructPublicMagicMetadata(publicMagicMetadataProps); - } - async encryptAsset( worker: Remote, file: FileWithMetadata, @@ -345,9 +330,9 @@ const uploadService = new UploadService(); export default uploadService; -export async function constructPublicMagicMetadata( +const constructPublicMagicMetadata = async ( publicMagicMetadataProps: FilePublicMagicMetadataProps, -): Promise { +): Promise => { const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps( publicMagicMetadataProps, ); @@ -356,7 +341,7 @@ export async function constructPublicMagicMetadata( return null; } return await updateMagicMetadata(publicMagicMetadataProps); -} +}; function getFileSize(file: File | ElectronFile) { return file.size; @@ -365,14 +350,19 @@ function getFileSize(file: File | ElectronFile) { export const getFileName = (file: File | ElectronFile | string) => typeof file == "string" ? basename(file) : file.name; +const readAsset = async ( + fileTypeInfo: FileTypeInfo, + { isLivePhoto, file, livePhotoAssets }: UploadAsset, +) => { + return isLivePhoto + ? await readLivePhoto(fileTypeInfo, livePhotoAssets) + : await readFile(fileTypeInfo, file); +}; + async function readFile( fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile, ): Promise { - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - rawFile, - fileTypeInfo, - ); log.info(`reading file data ${getFileNameSize(rawFile)} `); let filedata: Uint8Array | DataStream; if (!(rawFile instanceof File)) { @@ -390,8 +380,19 @@ async function readFile( filedata = await getUint8ArrayView(rawFile); } + if (filedata instanceof Uint8Array) { + + } else { + filedata.stream + } + log.info(`read file data successfully ${getFileNameSize(rawFile)} `); + const { thumbnail, hasStaticThumbnail } = await generateThumbnail( + rawFile, + fileTypeInfo, + ); + return { filedata, thumbnail, @@ -403,18 +404,19 @@ async function readLivePhoto( fileTypeInfo: FileTypeInfo, livePhotoAssets: LivePhotoAssets, ) { + const imageData = await getUint8ArrayView(livePhotoAssets.image); + + const videoData = await getUint8ArrayView(livePhotoAssets.video); + + const imageBlob = new Blob([imageData]); const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - livePhotoAssets.image, + imageBlob, { exactType: fileTypeInfo.imageType, fileType: FILE_TYPE.IMAGE, }, ); - const imageData = await getUint8ArrayView(livePhotoAssets.image); - - const videoData = await getUint8ArrayView(livePhotoAssets.video); - return { filedata: await encodeLivePhoto({ imageFileName: livePhotoAssets.image.name, @@ -649,17 +651,16 @@ export async function uploader( } log.info(`reading asset ${fileNameSize}`); - const file = await uploadService.readAsset(fileTypeInfo, uploadAsset); + const file = readAsset(fileTypeInfo, uploadAsset); if (file.hasStaticThumbnail) { metadata.hasStaticThumbnail = true; } - const pubMagicMetadata = - await uploadService.constructPublicMagicMetadata({ - ...publicMagicMetadata, - uploaderName, - }); + const pubMagicMetadata = await constructPublicMagicMetadata({ + ...publicMagicMetadata, + uploaderName, + }); const fileWithMetadata: FileWithMetadata = { localID, From 1f0c80cabc9b556cce5c3b7e5e0a03887d941c9a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 10:21:39 +0530 Subject: [PATCH 15/82] Refactor 1 --- desktop/src/main/ipc.ts | 8 +- desktop/src/main/services/ffmpeg.ts | 12 ++- desktop/src/main/services/image.ts | 21 ++-- desktop/src/preload.ts | 4 +- .../photos/src/services/upload/thumbnail.ts | 95 +++++++++---------- .../src/services/upload/uploadService.ts | 30 ++++++ web/apps/photos/src/types/upload/index.ts | 5 + web/packages/next/types/ipc.ts | 6 +- 8 files changed, 111 insertions(+), 70 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 9229d3b388..91efbb09f0 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -147,8 +147,12 @@ export const attachIPCHandlers = () => { ipcMain.handle( "generateImageThumbnail", - (_, imageData: Uint8Array, maxDimension: number, maxSize: number) => - generateImageThumbnail(imageData, maxDimension, maxSize), + ( + _, + dataOrPath: Uint8Array | string, + maxDimension: number, + maxSize: number, + ) => generateImageThumbnail(dataOrPath, maxDimension, maxSize), ); ipcMain.handle( diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index b1ae9e301a..f99f7ef8f1 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -48,17 +48,19 @@ export const ffmpegExec = async ( let inputFilePath: string; let isInputFileTemporary: boolean; - if (typeof dataOrPath == "string") { - inputFilePath = dataOrPath; - isInputFileTemporary = false; - } else { + if (dataOrPath instanceof Uint8Array) { inputFilePath = await makeTempFilePath(); isInputFileTemporary = true; - await fs.writeFile(inputFilePath, dataOrPath); + } else { + inputFilePath = dataOrPath; + isInputFileTemporary = false; } const outputFilePath = await makeTempFilePath(); try { + if (dataOrPath instanceof Uint8Array) + await fs.writeFile(inputFilePath, dataOrPath); + const cmd = substitutePlaceholders( command, inputFilePath, diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts index 47a1892449..7fae507578 100644 --- a/desktop/src/main/services/image.ts +++ b/desktop/src/main/services/image.ts @@ -63,14 +63,23 @@ const imageMagickPath = () => path.join(isDev ? "build" : process.resourcesPath, "image-magick"); export const generateImageThumbnail = async ( - imageData: Uint8Array, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ): Promise => { - const inputFilePath = await makeTempFilePath(); + let inputFilePath: string; + let isInputFileTemporary: boolean; + if (dataOrPath instanceof Uint8Array) { + inputFilePath = await makeTempFilePath(); + isInputFileTemporary = true; + } else { + inputFilePath = dataOrPath; + isInputFileTemporary = false; + } + const outputFilePath = await makeTempFilePath(".jpeg"); - // Construct the command first, it may throw NotAvailable on win32. + // Construct the command first, it may throw `NotAvailable` on win32. let quality = 70; let command = generateImageThumbnailCommand( inputFilePath, @@ -80,8 +89,8 @@ export const generateImageThumbnail = async ( ); try { - if (imageData instanceof Uint8Array) - await fs.writeFile(inputFilePath, imageData); + if (dataOrPath instanceof Uint8Array) + await fs.writeFile(inputFilePath, dataOrPath); let thumbnail: Uint8Array; do { @@ -98,7 +107,7 @@ export const generateImageThumbnail = async ( return thumbnail; } finally { try { - await deleteTempFile(inputFilePath); + if (isInputFileTemporary) await deleteTempFile(inputFilePath); await deleteTempFile(outputFilePath); } catch (e) { log.error("Ignoring error when cleaning up temp files", e); diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 0464022b00..728e8d012b 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -128,13 +128,13 @@ const convertToJPEG = (imageData: Uint8Array): Promise => ipcRenderer.invoke("convertToJPEG", imageData); const generateImageThumbnail = ( - imageData: Uint8Array, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ): Promise => ipcRenderer.invoke( "generateImageThumbnail", - imageData, + dataOrPath, maxDimension, maxSize, ); diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 6be2ea6bdf..aae7521e87 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,5 +1,5 @@ import log from "@/next/log"; -import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; +import { type Electron } from "@/next/types/ipc"; import { withTimeout } from "@ente/shared/utils"; import { FILE_TYPE } from "constants/file"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; @@ -13,30 +13,6 @@ const maxThumbnailDimension = 720; /** Maximum size (in bytes) of the generated thumbnail */ const maxThumbnailSize = 100 * 1024; // 100 KB -class ModuleState { - /** - * This will be set to true if we get an error from the Node.js side of our - * desktop app telling us that native JPEG conversion is not available for - * the current OS/arch combination. That way, we can stop pestering it again - * and again (saving an IPC round-trip). - * - * Note the double negative when it is used. - */ - isNativeThumbnailCreationNotAvailable = false; -} - -const moduleState = new ModuleState(); - -interface GeneratedThumbnail { - /** The JPEG data of the generated thumbnail */ - thumbnail: Uint8Array; - /** - * `true` if this is a fallback (all black) thumbnail we're returning since - * thumbnail generation failed for some reason. - */ - hasStaticThumbnail: boolean; -} - /** * Generate a JPEG thumbnail for the given image or video data. * @@ -46,19 +22,18 @@ interface GeneratedThumbnail { * itself that might not yet have support for more exotic formats. * * @param blob The data (blob) of the file whose thumbnail we want to generate. - * @param fileTypeInfo The type of the file whose {@link blob} we were given. + * @param fileTypeInfo The type information for the file. * - * @return {@link GeneratedThumbnail}, a thin wrapper for the raw JPEG bytes of - * the generated thumbnail. + * @return The JPEG data of the generated thumbnail. */ export const generateThumbnail = async ( blob: Blob, fileTypeInfo: FileTypeInfo, -): Promise => { +): Promise => { try { const thumbnail = fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnail(blob, fileTypeInfo) + ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) : await generateVideoThumbnail(blob); if (thumbnail.length == 0) throw new Error("Empty thumbnail"); @@ -73,38 +48,54 @@ export const generateThumbnail = async ( * A fallback, black, thumbnail for use in cases where thumbnail generation * fails. */ -const fallbackThumbnail = () => +export const fallbackThumbnail = () => Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); -const generateImageThumbnail = async ( - blob: Blob, +/** + * Generate a JPEG thumbnail for the given file using native tools. + * + * This function only works when we're running in the context of our desktop + * app, and this dependency is enforced by the need to pass the {@link electron} + * object which we use to perform IPC with the Node.js side of our desktop app. + * + * @param fileOrPath Either the image or video File, or the path to the image or + * video file on the user's local filesystem, whose thumbnail we want to + * generate. + * + * @param fileTypeInfo The type information for the file. + * + * @return The JPEG data of the generated thumbnail. + * + * @see {@link generateThumbnail}. + */ +export const generateThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, fileTypeInfo: FileTypeInfo, -) => { - const electron = globalThis.electron; - const available = !moduleState.isNativeThumbnailCreationNotAvailable; - if (electron && available) { - try { - return await generateImageThumbnailInElectron(electron, blob); - } catch (e) { - if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeThumbnailCreationNotAvailable = true; - } else { - log.error("Native thumbnail creation failed", e); - } - } - } +): Promise => { + try { + const thumbnail = + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailNative(electron, fileOrPath) + : await generateVideoThumbnail(blob); - return await generateImageThumbnailUsingCanvas(blob, fileTypeInfo); + if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + return { thumbnail, hasStaticThumbnail: false }; + } catch (e) { + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + } }; -const generateImageThumbnailInElectron = async ( +const generateImageThumbnailNative = async ( electron: Electron, - blob: Blob, + fileOrPath: File | string, ): Promise => { const startTime = Date.now(); - const data = new Uint8Array(await blob.arrayBuffer()); const jpegData = await electron.generateImageThumbnail( - data, + fileOrPath instanceof File + ? new Uint8Array(await fileOrPath.arrayBuffer()) + : fileOrPath, maxThumbnailDimension, maxThumbnailSize, ); diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index f2be02ec0f..2d16aab707 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -359,6 +359,22 @@ const readAsset = async ( : await readFile(fileTypeInfo, file); }; +// TODO(MR): Merge with the uploader +class ModuleState { + /** + * This will be set to true if we get an error from the Node.js side of our + * desktop app telling us that native JPEG conversion is not available for + * the current OS/arch combination. That way, we can stop pestering it again + * and again (saving an IPC round-trip). + * + * Note the double negative when it is used. + */ + isNativeThumbnailCreationNotAvailable = false; +} + +const moduleState = new ModuleState(); + + async function readFile( fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile, @@ -380,6 +396,20 @@ async function readFile( filedata = await getUint8ArrayView(rawFile); } + const electron = globalThis.electron; + const available = !moduleState.isNativeThumbnailCreationNotAvailable; + if (electron && available) { + try { + return await generateImageThumbnailInElectron(electron, blob); + } catch (e) { + if (e.message == CustomErrorMessage.NotAvailable) { + moduleState.isNativeThumbnailCreationNotAvailable = true; + } else { + log.error("Native thumbnail creation failed", e); + } + } + } + if (filedata instanceof Uint8Array) { } else { diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 175af6824f..f9d744dee7 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -114,7 +114,12 @@ export interface UploadURL { export interface FileInMemory { filedata: Uint8Array | DataStream; + /** The JPEG data of the generated thumbnail */ thumbnail: Uint8Array; + /** + * `true` if this is a fallback (all black) thumbnail we're returning since + * thumbnail generation failed for some reason. + */ hasStaticThumbnail: boolean; } diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 6ba5c78328..d392b6f3b4 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -221,8 +221,8 @@ export interface Electron { * not yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param imageData The raw image data (the contents of the image file) - * whose thumbnail we want to generate. + * @param dataOrPath The raw image data (the contents of the image file), or + * the path to the image file, whose thumbnail we want to generate. * @param maxDimension The maximum width or height of the generated * thumbnail. * @param maxSize Maximum size (in bytes) of the generated thumbnail. @@ -230,7 +230,7 @@ export interface Electron { * @returns JPEG data of the generated thumbnail. */ generateImageThumbnail: ( - imageData: Uint8Array, + dataOrPath: Uint8Array | string, maxDimension: number, maxSize: number, ) => Promise; From cd224001366b239ef8346a2643db8f02f75c45b1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 10:42:07 +0530 Subject: [PATCH 16/82] Agenda --- .../photos/src/services/upload/thumbnail.ts | 10 +-- .../src/services/upload/uploadService.ts | 68 ++++++++++++++++++- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index aae7521e87..b9a6482214 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -29,19 +29,11 @@ const maxThumbnailSize = 100 * 1024; // 100 KB export const generateThumbnail = async ( blob: Blob, fileTypeInfo: FileTypeInfo, -): Promise => { - try { - const thumbnail = +): Promise => fileTypeInfo.fileType === FILE_TYPE.IMAGE ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) : await generateVideoThumbnail(blob); - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } }; /** diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 2d16aab707..b2ca9c0639 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -375,7 +375,56 @@ class ModuleState { const moduleState = new ModuleState(); +/** + * Read the given file or path into an in-memory representation. + * + * [Note: The fileOrPath parameter to upload] + * + * The file can be either a web + * [File](https://developer.mozilla.org/en-US/docs/Web/API/File) or the absolute + * path to a file on desk. When and why, read on. + * + * This code gets invoked in two contexts: + * + * 1. web: the normal mode, when we're running in as a web app in the browser. + * + * 2. desktop: when we're running inside our desktop app. + * + * In the web context, we'll always get a File, since within the browser we + * cannot programmatically construct paths to or arbitrarily access files on the + * user's filesystem. Note that even if we were to have an absolute path at + * hand, we cannot programmatically create such File objects to arbitrary + * absolute paths on user's local filesystem for security reasons. + * + * So in the web context, this will always be a File we get as a result of an + * explicit user interaction (e.g. drag and drop). + * + * In the desktop context, this can be either a File or a path. + * + * 1. If the user provided us this file via some user interaction (say a drag + * and a drop), this'll still be a File. + * + * 2. However, when running in the desktop app we have the ability to access + * absolute paths on the user's file system. For example, if the user asks us + * to watch certain folders on their disk for changes, we'll be able to pick + * up new images being added, and in such cases, the parameter here will be a + * path. Another example is when resuming an previously interrupted upload - + * we'll only have the path at hand in such cases, not the File object. + * + * The advantage of the File object is that the browser has already read it into + * memory for us. The disadvantage comes in the case where we need to + * communicate with the native Node.js layer of our desktop app. Since this + * communication happens over IPC, the File's contents need to be serialized and + * copied, which is a bummer for large videos etc. + * + * So when we do have a path, we first try to see if we can perform IPC using + * the path itself (e.g. when generating thumbnails). Eventually, we'll need to + * read the file once when we need to encrypt and upload it, but if we're smart + * we can do all the rest of the IPC operations using the path itself, and for + * the read during upload using a streaming IPC mechanism. + */ async function readFile( + fileOrPath fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile, ): Promise { @@ -396,11 +445,13 @@ async function readFile( filedata = await getUint8ArrayView(rawFile); } + let thumbnail: Uint8Array + const electron = globalThis.electron; const available = !moduleState.isNativeThumbnailCreationNotAvailable; if (electron && available) { try { - return await generateImageThumbnailInElectron(electron, blob); + return await generateImageThumbnailNative(electron, fileOrPath); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { moduleState.isNativeThumbnailCreationNotAvailable = true; @@ -410,6 +461,21 @@ async function readFile( } } + + try { + const thumbnail = + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + : await generateVideoThumbnail(blob); + + if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + return { thumbnail, hasStaticThumbnail: false }; + } catch (e) { + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + } + + if (filedata instanceof Uint8Array) { } else { From 4a12774a3c26c77e90b16d6ea170aec07d793f67 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 11:07:10 +0530 Subject: [PATCH 17/82] Impl 1 --- web/apps/photos/src/services/ffmpeg.ts | 95 +++++++++---- .../photos/src/services/upload/thumbnail.ts | 132 +++++++++--------- .../src/services/upload/uploadService.ts | 37 ++--- web/packages/next/file.ts | 6 +- 4 files changed, 158 insertions(+), 112 deletions(-) diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 3c3f58a5fb..2778efb26f 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -1,4 +1,5 @@ import { ElectronFile } from "@/next/types/file"; +import type { Electron } from "@/next/types/ipc"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import { Remote } from "comlink"; @@ -12,31 +13,21 @@ import { ParsedExtractedMetadata } from "types/upload"; import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; /** - * Generate a thumbnail of the given video using FFmpeg. + * Generate a thumbnail for the given video using a wasm FFmpeg running in a web + * worker. * * This function is called during upload, when we need to generate thumbnails * for the new files that the user is adding. * * @param blob The input video blob. + * * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. */ -export const generateVideoThumbnail = async (blob: Blob) => { +export const generateVideoThumbnailWeb = async (blob: Blob) => { const thumbnailAtTime = (seekTime: number) => - ffmpegExec( - [ - ffmpegPathPlaceholder, - "-i", - inputPathPlaceholder, - "-ss", - `00:00:0${seekTime}`, - "-vframes", - "1", - "-vf", - "scale=-1:720", - outputPathPlaceholder, - ], - blob, - ); + ffmpegExecWeb(commandForThumbnailAtTime(seekTime), blob, 0); try { // Try generating thumbnail at seekTime 1 second. @@ -48,6 +39,50 @@ export const generateVideoThumbnail = async (blob: Blob) => { } }; +/** + * Generate a thumbnail for the given video using a native FFmpeg binary bundled + * with our desktop app. + * + * This function is called during upload, when we need to generate thumbnails + * for the new files that the user is adding. + * + * @param dataOrPath The input video's data or the path to the video on the + * user's local filesystem. See: [Note: The fileOrPath parameter to upload]. + * + * @returns JPEG data of the generated thumbnail. + * + * See also {@link generateVideoThumbnailNative}. + */ +export const generateVideoThumbnailNative = async ( + electron: Electron, + dataOrPath: Uint8Array | string, +) => { + const thumbnailAtTime = (seekTime: number) => + electron.ffmpegExec(commandForThumbnailAtTime(seekTime), dataOrPath, 0); + + try { + // Try generating thumbnail at seekTime 1 second. + return await thumbnailAtTime(1); + } catch (e) { + // If that fails, try again at the beginning. If even this throws, let + // it fail. + return await thumbnailAtTime(0); + } +}; + +const commandForThumbnailAtTime = (seekTime: number) => [ + ffmpegPathPlaceholder, + "-i", + inputPathPlaceholder, + "-ss", + `00:00:0${seekTime}`, + "-vframes", + "1", + "-vf", + "scale=-1:720", + outputPathPlaceholder, +]; + /** Called during upload */ export async function extractVideoMetadata(file: File | ElectronFile) { // https://stackoverflow.com/questions/9464617/retrieving-and-saving-media-metadata-using-ffmpeg @@ -157,16 +192,28 @@ export async function convertToMP4(file: File) { } /** - * Run the given FFmpeg command. - * - * If we're running in the context of our desktop app, use the FFmpeg binary we - * bundle with our desktop app to run the command. Otherwise fallback to using a - * wasm FFmpeg in a web worker. + * Run the given FFmpeg command using a wasm FFmpeg running in a web worker. * * As a rough ballpark, currently the native FFmpeg integration in the desktop * app is 10-20x faster than the wasm one. See: [Note: FFmpeg in Electron]. */ -const ffmpegExec = async ( +const ffmpegExecWeb = async ( + command: string[], + blob: Blob, + timeoutMs: number, +) => { + const worker = await workerFactory.lazy(); + return await worker.exec(command, blob, timeoutMs); +}; + +/** + * Run the given FFmpeg command using a native FFmpeg binary bundled with our + * desktop app. + * + * See also: {@link ffmpegExecWeb}. + */ +const ffmpegExecNative = async ( + electron: Electron, command: string[], blob: Blob, timeoutMs: number = 0, @@ -176,7 +223,7 @@ const ffmpegExec = async ( const data = new Uint8Array(await blob.arrayBuffer()); return await electron.ffmpegExec(command, data, timeoutMs); } else { - const worker = await workerFactory.lazy() + const worker = await workerFactory.lazy(); return await worker.exec(command, blob, timeoutMs); } }; diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index b9a6482214..01bb6c4f0c 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -30,79 +30,16 @@ export const generateThumbnail = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ): Promise => - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - : await generateVideoThumbnail(blob); - -}; - -/** - * A fallback, black, thumbnail for use in cases where thumbnail generation - * fails. - */ -export const fallbackThumbnail = () => - Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); - -/** - * Generate a JPEG thumbnail for the given file using native tools. - * - * This function only works when we're running in the context of our desktop - * app, and this dependency is enforced by the need to pass the {@link electron} - * object which we use to perform IPC with the Node.js side of our desktop app. - * - * @param fileOrPath Either the image or video File, or the path to the image or - * video file on the user's local filesystem, whose thumbnail we want to - * generate. - * - * @param fileTypeInfo The type information for the file. - * - * @return The JPEG data of the generated thumbnail. - * - * @see {@link generateThumbnail}. - */ -export const generateThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, - fileTypeInfo: FileTypeInfo, -): Promise => { - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailNative(electron, fileOrPath) - : await generateVideoThumbnail(blob); - - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } -}; - -const generateImageThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, -): Promise => { - const startTime = Date.now(); - const jpegData = await electron.generateImageThumbnail( - fileOrPath instanceof File - ? new Uint8Array(await fileOrPath.arrayBuffer()) - : fileOrPath, - maxThumbnailDimension, - maxThumbnailSize, - ); - log.debug( - () => `Native thumbnail generation took ${Date.now() - startTime} ms`, - ); - return jpegData; -}; + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + : await generateVideoThumbnail(blob); const generateImageThumbnailUsingCanvas = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ) => { if (isFileHEIC(fileTypeInfo.exactType)) { - log.debug(() => `Pre-converting ${fileTypeInfo.exactType} to JPEG`); + log.debug(() => `Pre-converting HEIC to JPEG for thumbnail generation`); blob = await heicToJPEG(blob); } @@ -234,3 +171,64 @@ const percentageSizeDiff = ( newThumbnailSize: number, oldThumbnailSize: number, ) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; + +/** + * A fallback, black, thumbnail for use in cases where thumbnail generation + * fails. + */ +export const fallbackThumbnail = () => + Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); + +/** + * Generate a JPEG thumbnail for the given file using native tools. + * + * This function only works when we're running in the context of our desktop + * app, and this dependency is enforced by the need to pass the {@link electron} + * object which we use to perform IPC with the Node.js side of our desktop app. + * + * @param fileOrPath Either the image or video File, or the path to the image or + * video file on the user's local filesystem, whose thumbnail we want to + * generate. + * + * @param fileTypeInfo The type information for the file. + * + * @return The JPEG data of the generated thumbnail. + * + * @see {@link generateThumbnail}. + */ +export const generateThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, +): Promise => { + try { + const thumbnail = + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailNative(electron, fileOrPath) + : await generateVideoThumbnail(blob); + + if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + return { thumbnail, hasStaticThumbnail: false }; + } catch (e) { + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + } +}; + +const generateImageThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, +): Promise => { + const startTime = Date.now(); + const jpegData = await electron.generateImageThumbnail( + fileOrPath instanceof File + ? new Uint8Array(await fileOrPath.arrayBuffer()) + : fileOrPath, + maxThumbnailDimension, + maxThumbnailSize, + ); + log.debug( + () => `Native thumbnail generation took ${Date.now() - startTime} ms`, + ); + return jpegData; +}; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b2ca9c0639..f49a2a1fb1 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -2,6 +2,7 @@ import { encodeLivePhoto } from "@/media/live-photo"; import { basename, convertBytesToHumanReadable, + fopLabel, getFileNameSize, } from "@/next/file"; import log from "@/next/log"; @@ -424,26 +425,10 @@ const moduleState = new ModuleState(); * the read during upload using a streaming IPC mechanism. */ async function readFile( - fileOrPath + fileOrPath: File | string, fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile, ): Promise { - log.info(`reading file data ${getFileNameSize(rawFile)} `); - let filedata: Uint8Array | DataStream; - if (!(rawFile instanceof File)) { - if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = await getElectronFileStream( - rawFile, - FILE_READER_CHUNK_SIZE, - ); - } else { - filedata = await getUint8ArrayView(rawFile); - } - } else if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getUint8ArrayView(rawFile); - } + log.info(`Reading file ${fopLabel(fileOrPath)} `); let thumbnail: Uint8Array @@ -461,6 +446,22 @@ async function readFile( } } + let filedata: Uint8Array | DataStream; + if (!(rawFile instanceof File)) { + if (rawFile.size > MULTIPART_PART_SIZE) { + filedata = await getElectronFileStream( + rawFile, + FILE_READER_CHUNK_SIZE, + ); + } else { + filedata = await getUint8ArrayView(rawFile); + } + } else if (rawFile.size > MULTIPART_PART_SIZE) { + filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); + } else { + filedata = await getUint8ArrayView(rawFile); + } + try { const thumbnail = diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index 4d05225c8e..02f936a18c 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -1,4 +1,4 @@ -import type { DesktopFilePath, ElectronFile } from "./types/file"; +import type { ElectronFile } from "./types/file"; /** * The two parts of a file name - the name itself, and an (optional) extension. @@ -70,8 +70,8 @@ export const dirname = (path: string) => { * Return a short description of the given {@link fileOrPath} suitable for * helping identify it in log messages. */ -export const fopLabel = (fileOrPath: File | DesktopFilePath) => - fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath.path; +export const fopLabel = (fileOrPath: File | string) => + fileOrPath instanceof File ? `File(${fileOrPath.name})` : fileOrPath; export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`; From 1f5fbcae76acd709766bca3d776bbc9348b7656a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 11:20:22 +0530 Subject: [PATCH 18/82] Checkpoint --- .../photos/src/services/upload/thumbnail.ts | 70 +++++++++---------- .../src/services/upload/uploadService.ts | 20 +++--- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index 01bb6c4f0c..b64521497d 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -14,25 +14,26 @@ const maxThumbnailDimension = 720; const maxThumbnailSize = 100 * 1024; // 100 KB /** - * Generate a JPEG thumbnail for the given image or video data. + * Generate a JPEG thumbnail for the given image or video blob. * * The thumbnail has a smaller file size so that is quick to load. But more * importantly, it uses a universal file format (JPEG in our case) so that the * thumbnail itself can be opened in all clients, even those like the web client * itself that might not yet have support for more exotic formats. * - * @param blob The data (blob) of the file whose thumbnail we want to generate. - * @param fileTypeInfo The type information for the file. + * @param blob The image or video blob whose thumbnail we want to generate. + * + * @param fileTypeInfo The type information for the file this blob came from. * * @return The JPEG data of the generated thumbnail. */ -export const generateThumbnail = async ( +export const generateThumbnailWeb = async ( blob: Blob, fileTypeInfo: FileTypeInfo, ): Promise => fileTypeInfo.fileType === FILE_TYPE.IMAGE ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - : await generateVideoThumbnail(blob); + : await generateVideoThumbnailWeb(blob); const generateImageThumbnailUsingCanvas = async ( blob: Blob, @@ -74,12 +75,12 @@ const generateImageThumbnailUsingCanvas = async ( return await compressedJPEGData(canvas); }; -const generateVideoThumbnail = async (blob: Blob) => { +const generateVideoThumbnailWeb = async (blob: Blob) => { try { - return await ffmpeg.generateVideoThumbnail(blob); + return await ffmpeg.generateVideoThumbnailWeb(blob); } catch (e) { log.error( - `Failed to generate video thumbnail using FFmpeg, will fallback to canvas`, + `Failed to generate video thumbnail using the wasm FFmpeg web worker, will fallback to canvas`, e, ); return generateVideoThumbnailUsingCanvas(blob); @@ -173,52 +174,35 @@ const percentageSizeDiff = ( ) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize; /** - * A fallback, black, thumbnail for use in cases where thumbnail generation - * fails. - */ -export const fallbackThumbnail = () => - Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); - -/** - * Generate a JPEG thumbnail for the given file using native tools. + * Generate a JPEG thumbnail for the given file or path using native tools. * * This function only works when we're running in the context of our desktop * app, and this dependency is enforced by the need to pass the {@link electron} * object which we use to perform IPC with the Node.js side of our desktop app. * - * @param fileOrPath Either the image or video File, or the path to the image or - * video file on the user's local filesystem, whose thumbnail we want to + * @param dataOrPath The image or video {@link File}, or the path to the image + * or video file on the user's local filesystem, whose thumbnail we want to * generate. * - * @param fileTypeInfo The type information for the file. + * @param fileTypeInfo The type information for {@link fileOrPath}. * * @return The JPEG data of the generated thumbnail. * - * @see {@link generateThumbnail}. + * See also {@link generateThumbnailWeb}. */ export const generateThumbnailNative = async ( electron: Electron, fileOrPath: File | string, fileTypeInfo: FileTypeInfo, -): Promise => { - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailNative(electron, fileOrPath) - : await generateVideoThumbnail(blob); - - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } -}; +): Promise => + fileTypeInfo.fileType === FILE_TYPE.IMAGE + ? await generateImageThumbnailNative(electron, fileOrPath) + : await generateVideoThumbnailNative(blob); const generateImageThumbnailNative = async ( electron: Electron, fileOrPath: File | string, -): Promise => { +) => { const startTime = Date.now(); const jpegData = await electron.generateImageThumbnail( fileOrPath instanceof File @@ -232,3 +216,19 @@ const generateImageThumbnailNative = async ( ); return jpegData; }; + +const dataOrPath = (fileOrPath) => { + fileOrPath +} +const generateVideoThumbnailNative = async ( + electron: Electron, + fileOrPath: File | string, +) => ffmpeg.generateVideoThumbnailNative(electron, ) + + +/** + * A fallback, black, thumbnail for use in cases where thumbnail generation + * fails. + */ +export const fallbackThumbnail = () => + Uint8Array.from(atob(BLACK_THUMBNAIL_BASE64), (c) => c.charCodeAt(0)); diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index f49a2a1fb1..f531c22388 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -375,7 +375,6 @@ class ModuleState { const moduleState = new ModuleState(); - /** * Read the given file or path into an in-memory representation. * @@ -424,13 +423,19 @@ const moduleState = new ModuleState(); * we can do all the rest of the IPC operations using the path itself, and for * the read during upload using a streaming IPC mechanism. */ -async function readFile( +const readFileOrPath = async ( fileOrPath: File | string, fileTypeInfo: FileTypeInfo, -): Promise { +): Promise => { log.info(`Reading file ${fopLabel(fileOrPath)} `); - let thumbnail: Uint8Array + // If it's a file, read it into data + const dataOrPath = + fileOrPath instanceof File + ? new Uint8Array(await fileOrPath.arrayBuffer()) + : fileOrPath; + + let thumbnail: Uint8Array; const electron = globalThis.electron; const available = !moduleState.isNativeThumbnailCreationNotAvailable; @@ -462,7 +467,6 @@ async function readFile( filedata = await getUint8ArrayView(rawFile); } - try { const thumbnail = fileTypeInfo.fileType === FILE_TYPE.IMAGE @@ -476,11 +480,9 @@ async function readFile( return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; } - if (filedata instanceof Uint8Array) { - } else { - filedata.stream + filedata.stream; } log.info(`read file data successfully ${getFileNameSize(rawFile)} `); @@ -495,7 +497,7 @@ async function readFile( thumbnail, hasStaticThumbnail, }; -} +}; async function readLivePhoto( fileTypeInfo: FileTypeInfo, From 76be5e37d52ac12323733ea77aec6aec8fb5be41 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 11:32:40 +0530 Subject: [PATCH 19/82] cp --- .../photos/src/services/upload/thumbnail.ts | 45 +++++-------------- .../src/services/upload/uploadService.ts | 17 ++++--- web/apps/photos/src/utils/file/index.ts | 6 ++- 3 files changed, 25 insertions(+), 43 deletions(-) diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index b64521497d..e2eccf9b89 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -180,11 +180,11 @@ const percentageSizeDiff = ( * app, and this dependency is enforced by the need to pass the {@link electron} * object which we use to perform IPC with the Node.js side of our desktop app. * - * @param dataOrPath The image or video {@link File}, or the path to the image - * or video file on the user's local filesystem, whose thumbnail we want to - * generate. + * @param dataOrPath Contents of an image or video file, or the path to the + * image or video file on the user's local filesystem, whose thumbnail we want + * to generate. * - * @param fileTypeInfo The type information for {@link fileOrPath}. + * @param fileTypeInfo The type information for {@link dataOrPath}. * * @return The JPEG data of the generated thumbnail. * @@ -192,39 +192,16 @@ const percentageSizeDiff = ( */ export const generateThumbnailNative = async ( electron: Electron, - fileOrPath: File | string, + dataOrPath: Uint8Array | string, fileTypeInfo: FileTypeInfo, ): Promise => fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailNative(electron, fileOrPath) - : await generateVideoThumbnailNative(blob); - -const generateImageThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, -) => { - const startTime = Date.now(); - const jpegData = await electron.generateImageThumbnail( - fileOrPath instanceof File - ? new Uint8Array(await fileOrPath.arrayBuffer()) - : fileOrPath, - maxThumbnailDimension, - maxThumbnailSize, - ); - log.debug( - () => `Native thumbnail generation took ${Date.now() - startTime} ms`, - ); - return jpegData; -}; - -const dataOrPath = (fileOrPath) => { - fileOrPath -} -const generateVideoThumbnailNative = async ( - electron: Electron, - fileOrPath: File | string, -) => ffmpeg.generateVideoThumbnailNative(electron, ) - + ? await electron.generateImageThumbnail( + dataOrPath, + maxThumbnailDimension, + maxThumbnailSize, + ) + : ffmpeg.generateVideoThumbnailNative(electron, dataOrPath); /** * A fallback, black, thumbnail for use in cases where thumbnail generation diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index f531c22388..c0ec389743 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -364,13 +364,15 @@ const readAsset = async ( class ModuleState { /** * This will be set to true if we get an error from the Node.js side of our - * desktop app telling us that native JPEG conversion is not available for - * the current OS/arch combination. That way, we can stop pestering it again - * and again (saving an IPC round-trip). + * desktop app telling us that native image thumbnail generation is not + * available for the current OS/arch combination. + * + * That way, we can stop pestering it again and again (saving an IPC + * round-trip). * * Note the double negative when it is used. */ - isNativeThumbnailCreationNotAvailable = false; + isNativeImageThumbnailCreationNotAvailable = false; } const moduleState = new ModuleState(); @@ -429,7 +431,8 @@ const readFileOrPath = async ( ): Promise => { log.info(`Reading file ${fopLabel(fileOrPath)} `); - // If it's a file, read it into data + // If it's a file, read-in its data. We need to do it once anyway for + // generating the thumbnail. const dataOrPath = fileOrPath instanceof File ? new Uint8Array(await fileOrPath.arrayBuffer()) @@ -438,8 +441,8 @@ const readFileOrPath = async ( let thumbnail: Uint8Array; const electron = globalThis.electron; - const available = !moduleState.isNativeThumbnailCreationNotAvailable; - if (electron && available) { + if (electron) { + if !moduleState.isNativeImageThumbnailCreationNotAvailable; try { return await generateImageThumbnailNative(electron, fileOrPath); } catch (e) { diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index a6cb640b6a..c37644b700 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -62,8 +62,10 @@ class ModuleState { /** * This will be set to true if we get an error from the Node.js side of our * desktop app telling us that native JPEG conversion is not available for - * the current OS/arch combination. That way, we can stop pestering it again - * and again (saving an IPC round-trip). + * the current OS/arch combination. + * + * That way, we can stop pestering it again and again (saving an IPC + * round-trip). * * Note the double negative when it is used. */ From 1d4efd738ca1a23f2b06438bcfd8e5ffe17c47ab Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 12:03:55 +0530 Subject: [PATCH 20/82] Stream reader --- desktop/src/main/stream.ts | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/desktop/src/main/stream.ts b/desktop/src/main/stream.ts index 8ddb80dc6a..14447e7cb3 100644 --- a/desktop/src/main/stream.ts +++ b/desktop/src/main/stream.ts @@ -1,15 +1,16 @@ /** * @file stream data to-from renderer using a custom protocol handler. */ -import { protocol } from "electron/main"; +import { net, protocol } from "electron/main"; import { createWriteStream, existsSync } from "node:fs"; import fs from "node:fs/promises"; import { Readable } from "node:stream"; +import { pathToFileURL } from "node:url"; import log from "./log"; /** * Register a protocol handler that we use for streaming large files between the - * main process (node) and the renderer process (browser) layer. + * main (Node.js) and renderer (Chromium) processes. * * [Note: IPC streams] * @@ -17,11 +18,14 @@ import log from "./log"; * across IPC. And passing the entire contents of the file is not feasible for * large video files because of the memory pressure the copying would entail. * - * As an alternative, we register a custom protocol handler that can provided a + * As an alternative, we register a custom protocol handler that can provides a * bi-directional stream. The renderer can stream data to the node side by * streaming the request. The node side can stream to the renderer side by * streaming the response. * + * The stream is not full duplex - while both reads and writes can be streamed, + * they need to be streamed separately. + * * See also: [Note: Transferring large amount of data over IPC] * * Depends on {@link registerPrivilegedSchemes}. @@ -29,12 +33,16 @@ import log from "./log"; export const registerStreamProtocol = () => { protocol.handle("stream", async (request: Request) => { const url = request.url; + // The request URL contains the command to run as the host, and the + // pathname of the file as the path. For example, + // + // stream://write/path/to/file + // host-pathname----- + // const { host, pathname } = new URL(url); // Convert e.g. "%20" to spaces. const path = decodeURIComponent(pathname); switch (host) { - /* stream://write/path/to/file */ - /* host-pathname----- */ case "write": try { await writeStream(path, request.body); @@ -46,6 +54,17 @@ export const registerStreamProtocol = () => { { status: 500 }, ); } + + case "read": + try { + return net.fetch(pathToFileURL(path).toString()); + } catch (e) { + log.error(`Failed to read stream for ${url}`, e); + return new Response(`Failed to read stream: ${e.message}`, { + status: 500, + }); + } + default: return new Response("", { status: 404 }); } From cb0d25030ddf024f9cfc335d732c73b7a14f2bfb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 12:10:46 +0530 Subject: [PATCH 21/82] API 1 --- desktop/src/main/stream.ts | 20 ++++++++-------- web/apps/photos/src/utils/native-stream.ts | 28 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/desktop/src/main/stream.ts b/desktop/src/main/stream.ts index 14447e7cb3..d6768069fb 100644 --- a/desktop/src/main/stream.ts +++ b/desktop/src/main/stream.ts @@ -43,6 +43,16 @@ export const registerStreamProtocol = () => { // Convert e.g. "%20" to spaces. const path = decodeURIComponent(pathname); switch (host) { + case "read": + try { + return net.fetch(pathToFileURL(path).toString()); + } catch (e) { + log.error(`Failed to read stream for ${url}`, e); + return new Response(`Failed to read stream: ${e.message}`, { + status: 500, + }); + } + case "write": try { await writeStream(path, request.body); @@ -55,16 +65,6 @@ export const registerStreamProtocol = () => { ); } - case "read": - try { - return net.fetch(pathToFileURL(path).toString()); - } catch (e) { - log.error(`Failed to read stream for ${url}`, e); - return new Response(`Failed to read stream: ${e.message}`, { - status: 500, - }); - } - default: return new Response("", { status: 404 }); } diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index 7dba1acf9c..742c00a611 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -4,6 +4,34 @@ * NOTE: These functions only work when we're running in our desktop app. */ +/** + * Stream the given file from the user's local filesystem. + * + * **This only works when we're running in our desktop app**. It uses the + * "stream://" protocol handler exposed by our custom code in the Node.js layer. + * See: [Note: IPC streams]. + * + * @param path The path on the file on the user's local filesystem whose + * contents we want to stream. + * + * @return A standard web {@link Response} object that contains the contents of + * the file. In particular, `response.body` will be a {@link ReadableStream} + * that can be used to read the files contents in a streaming, chunked, manner. + */ +export const readStream = async (path: string) => { + const req = new Request(`stream://read${path}`, { + method: "GET", + }); + + const res = await fetch(req); + if (!res.ok) + throw new Error( + `Failed to read stream from ${path}: HTTP ${res.status}`, + ); + + return res; +}; + /** * Write the given stream to a file on the local machine. * From e6e235490a7e6487dd04d8270426c8e51ae71ba2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 12:35:22 +0530 Subject: [PATCH 22/82] Content-Length --- desktop/src/main/stream.ts | 56 ++++++++++++++-------- web/apps/photos/src/utils/native-stream.ts | 2 + 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/desktop/src/main/stream.ts b/desktop/src/main/stream.ts index d6768069fb..12db786f53 100644 --- a/desktop/src/main/stream.ts +++ b/desktop/src/main/stream.ts @@ -44,33 +44,49 @@ export const registerStreamProtocol = () => { const path = decodeURIComponent(pathname); switch (host) { case "read": - try { - return net.fetch(pathToFileURL(path).toString()); - } catch (e) { - log.error(`Failed to read stream for ${url}`, e); - return new Response(`Failed to read stream: ${e.message}`, { - status: 500, - }); - } - + return handleRead(path); case "write": - try { - await writeStream(path, request.body); - return new Response("", { status: 200 }); - } catch (e) { - log.error(`Failed to write stream for ${url}`, e); - return new Response( - `Failed to write stream: ${e.message}`, - { status: 500 }, - ); - } - + return handleWrite(path, request); default: return new Response("", { status: 404 }); } }); }; +const handleRead = async (path: string) => { + try { + const res = await net.fetch(pathToFileURL(path).toString()); + if (res.ok) { + // net.fetch defaults to text/plain, which might be fine + // in practice, but as an extra precaution indicate that + // this is binary data. + res.headers.set("Content-Type", "application/octet-stream"); + + // Add the file's size as the Content-Length header. + const fileSize = (await fs.stat(path)).size; + res.headers.set("Content-Length", `${fileSize}`); + } + return res; + } catch (e) { + log.error(`Failed to read stream at ${path}`, e); + return new Response(`Failed to read stream: ${e.message}`, { + status: 500, + }); + } +}; + +const handleWrite = async (path: string, request: Request) => { + try { + await writeStream(path, request.body); + return new Response("", { status: 200 }); + } catch (e) { + log.error(`Failed to write stream to ${path}`, e); + return new Response(`Failed to write stream: ${e.message}`, { + status: 500, + }); + } +}; + /** * Write a (web) ReadableStream to a file at the given {@link filePath}. * diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index 742c00a611..b98e6ae1a2 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -17,6 +17,8 @@ * @return A standard web {@link Response} object that contains the contents of * the file. In particular, `response.body` will be a {@link ReadableStream} * that can be used to read the files contents in a streaming, chunked, manner. + * Also, the response is guaranteed to have a "Content-Length" header indicating + * the size of the file that we'll be reading from disk. */ export const readStream = async (path: string) => { const req = new Request(`stream://read${path}`, { From a286b11adba1e01967a2ef3fcebf87b7b7cf3157 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 12:55:27 +0530 Subject: [PATCH 23/82] Checkpoint --- .../src/services/upload/uploadService.ts | 104 +++++++++--------- web/apps/photos/src/utils/native-stream.ts | 18 ++- 2 files changed, 66 insertions(+), 56 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index c0ec389743..f3e8b852e9 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -54,12 +54,9 @@ import { getNonEmptyMagicMetadataProps, updateMagicMetadata, } from "utils/magicMetadata"; +import { readStream } from "utils/native-stream"; import { findMatchingExistingFiles } from "utils/upload"; -import { - getElectronFileStream, - getFileStream, - getUint8ArrayView, -} from "../readerService"; +import { getFileStream, getUint8ArrayView } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, @@ -431,6 +428,29 @@ const readFileOrPath = async ( ): Promise => { log.info(`Reading file ${fopLabel(fileOrPath)} `); + let dataOrStream: Uint8Array | DataStream; + if (fileOrPath instanceof File) { + const file = fileOrPath; + if (file.size > MULTIPART_PART_SIZE) { + dataOrStream = getFileStream(file, FILE_READER_CHUNK_SIZE); + } else { + dataOrStream = new Uint8Array(await file.arrayBuffer()); + } + } else { + const path = fileOrPath; + const { stream, size } = await readStream(path); + if (size > MULTIPART_PART_SIZE) { + const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); + dataOrStream = { stream, chunkCount }; + } else { + dataOrStream = new Uint8Array( + await new Response(stream).arrayBuffer(), + ); + } + } + + let filedata: Uint8Array | DataStream; + // If it's a file, read-in its data. We need to do it once anyway for // generating the thumbnail. const dataOrPath = @@ -438,55 +458,39 @@ const readFileOrPath = async ( ? new Uint8Array(await fileOrPath.arrayBuffer()) : fileOrPath; - let thumbnail: Uint8Array; + // let thumbnail: Uint8Array; - const electron = globalThis.electron; - if (electron) { - if !moduleState.isNativeImageThumbnailCreationNotAvailable; - try { - return await generateImageThumbnailNative(electron, fileOrPath); - } catch (e) { - if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeThumbnailCreationNotAvailable = true; - } else { - log.error("Native thumbnail creation failed", e); - } - } - } + // const electron = globalThis.electron; + // if (electron) { + // if !moduleState.isNativeImageThumbnailCreationNotAvailable; + // try { + // return await generateImageThumbnailNative(electron, fileOrPath); + // } catch (e) { + // if (e.message == CustomErrorMessage.NotAvailable) { + // moduleState.isNativeThumbnailCreationNotAvailable = true; + // } else { + // log.error("Native thumbnail creation failed", e); + // } + // } + // } - let filedata: Uint8Array | DataStream; - if (!(rawFile instanceof File)) { - if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = await getElectronFileStream( - rawFile, - FILE_READER_CHUNK_SIZE, - ); - } else { - filedata = await getUint8ArrayView(rawFile); - } - } else if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getUint8ArrayView(rawFile); - } + // try { + // const thumbnail = + // fileTypeInfo.fileType === FILE_TYPE.IMAGE + // ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + // : await generateVideoThumbnail(blob); - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - : await generateVideoThumbnail(blob); + // if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + // return { thumbnail, hasStaticThumbnail: false }; + // } catch (e) { + // log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + // return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + // } - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } - - if (filedata instanceof Uint8Array) { - } else { - filedata.stream; - } + // if (filedata instanceof Uint8Array) { + // } else { + // filedata.stream; + // } log.info(`read file data successfully ${getFileNameSize(rawFile)} `); diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index b98e6ae1a2..aef06845c0 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -14,13 +14,13 @@ * @param path The path on the file on the user's local filesystem whose * contents we want to stream. * - * @return A standard web {@link Response} object that contains the contents of - * the file. In particular, `response.body` will be a {@link ReadableStream} - * that can be used to read the files contents in a streaming, chunked, manner. - * Also, the response is guaranteed to have a "Content-Length" header indicating + * @return A (ReadableStream, size) tuple. The {@link ReadableStream} can be + * used to read the files contents in a streaming manner. The size value is the * the size of the file that we'll be reading from disk. */ -export const readStream = async (path: string) => { +export const readStream = async ( + path: string, +): Promise<{ stream: ReadableStream; size: number }> => { const req = new Request(`stream://read${path}`, { method: "GET", }); @@ -31,7 +31,13 @@ export const readStream = async (path: string) => { `Failed to read stream from ${path}: HTTP ${res.status}`, ); - return res; + const size = +res.headers["Content-Length"]; + if (isNaN(size)) + throw new Error( + `Got a numeric Content-Length when reading a stream. The response was ${res}`, + ); + + return { stream: res.body, size }; }; /** From 66c64d0c58f6b855c1456d5913df76bc8fa6f3be Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 13:07:27 +0530 Subject: [PATCH 24/82] Let the caller decide --- .../photos/src/services/upload/uploadService.ts | 8 +++----- web/apps/photos/src/utils/native-stream.ts | 14 +++++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index f3e8b852e9..6bfbcb1034 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -438,14 +438,12 @@ const readFileOrPath = async ( } } else { const path = fileOrPath; - const { stream, size } = await readStream(path); + const { response, size } = await readStream(path); if (size > MULTIPART_PART_SIZE) { const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); - dataOrStream = { stream, chunkCount }; + dataOrStream = { stream: response.body, chunkCount }; } else { - dataOrStream = new Uint8Array( - await new Response(stream).arrayBuffer(), - ); + dataOrStream = new Uint8Array(await response.arrayBuffer()); } } diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index aef06845c0..52da84f99d 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -14,13 +14,17 @@ * @param path The path on the file on the user's local filesystem whose * contents we want to stream. * - * @return A (ReadableStream, size) tuple. The {@link ReadableStream} can be - * used to read the files contents in a streaming manner. The size value is the - * the size of the file that we'll be reading from disk. + * @return A ({@link Response}, size) tuple. + * + * * The response contains the contents of the file. In particular, the `body` + * {@link ReadableStream} property of this response can be used to read the + * files contents in a streaming manner. + * + * * The size is the size of the file that we'll be reading from disk. */ export const readStream = async ( path: string, -): Promise<{ stream: ReadableStream; size: number }> => { +): Promise<{ response: Response; size: number }> => { const req = new Request(`stream://read${path}`, { method: "GET", }); @@ -37,7 +41,7 @@ export const readStream = async ( `Got a numeric Content-Length when reading a stream. The response was ${res}`, ); - return { stream: res.body, size }; + return { response: res, size }; }; /** From abbfbf695f1578417df0dd3894379750b36cb865 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 13:13:25 +0530 Subject: [PATCH 25/82] Split earlier --- .../src/services/upload/uploadService.ts | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 6bfbcb1034..4a5ee37df3 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -70,7 +70,7 @@ import { } from "./metadataService"; import { uploadStreamUsingMultipart } from "./multiPartUploadService"; import publicUploadHttpClient from "./publicUploadHttpClient"; -import { generateThumbnail } from "./thumbnail"; +import { fallbackThumbnail, generateThumbnailWeb } from "./thumbnail"; import UIService from "./uiService"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; @@ -422,6 +422,39 @@ const moduleState = new ModuleState(); * we can do all the rest of the IPC operations using the path itself, and for * the read during upload using a streaming IPC mechanism. */ +const readFileOrPath = async ( + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, +): Promise => + fileOrPath instanceof File + ? _readFile(fileOrPath, fileTypeInfo) + : _readPath(fileOrPath, fileTypeInfo); + +const _readFile = async (file: File, fileTypeInfo: FileTypeInfo) => { + const dataOrStream = + file.size > MULTIPART_PART_SIZE + ? getFileStream(file, FILE_READER_CHUNK_SIZE) + : new Uint8Array(await file.arrayBuffer()); + + let thumbnail: Uint8Array; + let hasStaticThumbnail = false; + + try { + thumbnail = await generateThumbnailWeb(file, fileTypeInfo); + if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + } catch (e) { + log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + thumbnail = fallbackThumbnail(); + hasStaticThumbnail = true; + } + + return { + filedata: dataOrStream, + thumbnail, + hasStaticThumbnail, + }; +}; + const readFileOrPath = async ( fileOrPath: File | string, fileTypeInfo: FileTypeInfo, From 6ca3eb55af149cf539ee1903926ee163678491ec Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 13:49:09 +0530 Subject: [PATCH 26/82] Try another factoring --- .../src/services/upload/uploadService.ts | 135 +++++++++++------- 1 file changed, 82 insertions(+), 53 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 4a5ee37df3..72e8ed1f35 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,12 +1,13 @@ import { encodeLivePhoto } from "@/media/live-photo"; +import { ensureElectron } from "@/next/electron"; import { basename, convertBytesToHumanReadable, - fopLabel, getFileNameSize, } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; +import { CustomErrorMessage } from "@/next/types/ipc"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { B64EncryptionResult, @@ -70,7 +71,11 @@ import { } from "./metadataService"; import { uploadStreamUsingMultipart } from "./multiPartUploadService"; import publicUploadHttpClient from "./publicUploadHttpClient"; -import { fallbackThumbnail, generateThumbnailWeb } from "./thumbnail"; +import { + fallbackThumbnail, + generateThumbnailNative, + generateThumbnailWeb, +} from "./thumbnail"; import UIService from "./uiService"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; @@ -441,7 +446,6 @@ const _readFile = async (file: File, fileTypeInfo: FileTypeInfo) => { try { thumbnail = await generateThumbnailWeb(file, fileTypeInfo); - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); } catch (e) { log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); thumbnail = fallbackThumbnail(); @@ -455,56 +459,92 @@ const _readFile = async (file: File, fileTypeInfo: FileTypeInfo) => { }; }; -const readFileOrPath = async ( - fileOrPath: File | string, - fileTypeInfo: FileTypeInfo, -): Promise => { - log.info(`Reading file ${fopLabel(fileOrPath)} `); +const _readPath = async (path: string, fileTypeInfo: FileTypeInfo) => { + const electron = ensureElectron(); let dataOrStream: Uint8Array | DataStream; - if (fileOrPath instanceof File) { - const file = fileOrPath; - if (file.size > MULTIPART_PART_SIZE) { - dataOrStream = getFileStream(file, FILE_READER_CHUNK_SIZE); - } else { - dataOrStream = new Uint8Array(await file.arrayBuffer()); - } + + const { response, size } = await readStream(path); + if (size > MULTIPART_PART_SIZE) { + const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); + dataOrStream = { stream: response.body, chunkCount }; } else { - const path = fileOrPath; - const { response, size } = await readStream(path); - if (size > MULTIPART_PART_SIZE) { - const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); - dataOrStream = { stream: response.body, chunkCount }; + dataOrStream = new Uint8Array(await response.arrayBuffer()); + } + + let thumbnail: Uint8Array | undefined; + let hasStaticThumbnail = false; + + // On Windows native thumbnail creation for images is not yet implemented. + const notAvailable = + fileTypeInfo.fileType == FILE_TYPE.IMAGE && + moduleState.isNativeImageThumbnailCreationNotAvailable; + + try { + if (!notAvailable) { + thumbnail = await generateThumbnailNative( + electron, + path, + fileTypeInfo, + ); + } + } catch (e) { + if (e.message == CustomErrorMessage.NotAvailable) { + moduleState.isNativeImageThumbnailCreationNotAvailable = true; } else { - dataOrStream = new Uint8Array(await response.arrayBuffer()); + log.error("Native thumbnail creation failed", e); } } - let filedata: Uint8Array | DataStream; + // If needed, fallback to browser based thumbnail generation. + if (!thumbnail) { + let data: Uint8Array | undefined; + if (dataOrStream instanceof Uint8Array) { + data = dataOrStream; + } else { + // Read the stream into memory, since the our web based thumbnail + // generation methods need the entire file in memory. Don't try this + // fallback for huge files though lest we run out of memory. + if (size < 100 * 1024 * 1024 /* 100 MB */) { + data = new Uint8Array( + await new Response(dataOrStream.stream).arrayBuffer(), + ); + // The Readable stream cannot be read twice, so also overwrite + // the stream with the data we read. + dataOrStream = data; + } + } + if (data) { + const blob = new Blob([data]); + try { + thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); + } catch (e) { + log.error( + `Failed to generate ${fileTypeInfo.exactType} thumbnail`, + e, + ); + } + } + } - // If it's a file, read-in its data. We need to do it once anyway for - // generating the thumbnail. - const dataOrPath = - fileOrPath instanceof File - ? new Uint8Array(await fileOrPath.arrayBuffer()) - : fileOrPath; + if (!thumbnail) { + thumbnail = fallbackThumbnail(); + hasStaticThumbnail = true; + } + + return { + filedata: dataOrStream, + thumbnail, + hasStaticThumbnail, + }; + + // const dataOrPath = + // fileOrPath instanceof File + // ? new Uint8Array(await fileOrPath.arrayBuffer()) + // : fileOrPath; // let thumbnail: Uint8Array; - // const electron = globalThis.electron; - // if (electron) { - // if !moduleState.isNativeImageThumbnailCreationNotAvailable; - // try { - // return await generateImageThumbnailNative(electron, fileOrPath); - // } catch (e) { - // if (e.message == CustomErrorMessage.NotAvailable) { - // moduleState.isNativeThumbnailCreationNotAvailable = true; - // } else { - // log.error("Native thumbnail creation failed", e); - // } - // } - // } - // try { // const thumbnail = // fileTypeInfo.fileType === FILE_TYPE.IMAGE @@ -524,17 +564,6 @@ const readFileOrPath = async ( // } log.info(`read file data successfully ${getFileNameSize(rawFile)} `); - - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - rawFile, - fileTypeInfo, - ); - - return { - filedata, - thumbnail, - hasStaticThumbnail, - }; }; async function readLivePhoto( From 6ff41db9396880fef50d5896b457d337df03ca42 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 14:16:41 +0530 Subject: [PATCH 27/82] Try another factoring --- web/apps/photos/src/services/export/index.ts | 3 + .../src/services/upload/uploadService.ts | 120 +++++++++++++++++- web/apps/photos/src/utils/file/index.ts | 18 ++- web/apps/photos/src/utils/native-stream.ts | 22 +++- 4 files changed, 152 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index dc7d40c70c..0bf355d8d7 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -994,6 +994,7 @@ class ExportService { file, ); await writeStream( + electron, `${collectionExportPath}/${fileExportName}`, updatedFileStream, ); @@ -1047,6 +1048,7 @@ class ExportService { file, ); await writeStream( + electron, `${collectionExportPath}/${imageExportName}`, imageStream, ); @@ -1061,6 +1063,7 @@ class ExportService { ); try { await writeStream( + electron, `${collectionExportPath}/${videoExportName}`, videoStream, ); diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 72e8ed1f35..6ec2d370f1 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -430,10 +430,122 @@ const moduleState = new ModuleState(); const readFileOrPath = async ( fileOrPath: File | string, fileTypeInfo: FileTypeInfo, -): Promise => - fileOrPath instanceof File - ? _readFile(fileOrPath, fileTypeInfo) - : _readPath(fileOrPath, fileTypeInfo); +): Promise => { + let file: File | undefined; + let dataOrStream: Uint8Array | DataStream; + let fileSize: number; + + if (fileOrPath instanceof File) { + file = fileOrPath; + fileSize = file.size; + dataOrStream = + fileSize > MULTIPART_PART_SIZE + ? getFileStream(file, FILE_READER_CHUNK_SIZE) + : new Uint8Array(await file.arrayBuffer()); + } else { + const path = fileOrPath; + const { response, size } = await readStream(ensureElectron(), path); + fileSize = size; + if (size > MULTIPART_PART_SIZE) { + const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); + dataOrStream = { stream: response.body, chunkCount }; + } else { + dataOrStream = new Uint8Array(await response.arrayBuffer()); + } + } + + let thumbnail: Uint8Array | undefined; + let hasStaticThumbnail = false; + + const electron = globalThis.electron; + if (electron) { + // On Windows native thumbnail creation for images is not yet implemented. + const notAvailable = + fileTypeInfo.fileType == FILE_TYPE.IMAGE && + moduleState.isNativeImageThumbnailCreationNotAvailable; + + try { + if (!notAvailable) { + if (fileOrPath instanceof File) { + if (dataOrStream instanceof Uint8Array) { + thumbnail = await generateThumbnailNative( + electron, + dataOrStream, + fileTypeInfo, + ); + } + } else { + thumbnail = await generateThumbnailNative( + electron, + fileOrPath, + fileTypeInfo, + ); + } + } + } catch (e) { + if (e.message == CustomErrorMessage.NotAvailable) { + moduleState.isNativeImageThumbnailCreationNotAvailable = true; + } else { + log.error("Native thumbnail creation failed", e); + } + } + } + + // If needed, fallback to browser based thumbnail generation. First, see if + // we already have a file (which is also a blob). + if (!thumbnail && file) { + try { + thumbnail = await generateThumbnailWeb(file, fileTypeInfo); + } catch (e) { + log.error( + `Failed to generate ${fileTypeInfo.exactType} thumbnail`, + e, + ); + } + } + + // Otherwise see if the data is small enough to read in memory. + if (!thumbnail) { + let data: Uint8Array | undefined; + if (dataOrStream instanceof Uint8Array) { + data = dataOrStream; + } else { + // Read the stream into memory, since the our web based thumbnail + // generation methods need the entire file in memory. Don't try this + // fallback for huge files though lest we run out of memory. + if (fileSize < 100 * 1024 * 1024 /* 100 MB */) { + data = new Uint8Array( + await new Response(dataOrStream.stream).arrayBuffer(), + ); + // The Readable stream cannot be read twice, so also overwrite + // the stream with the data we read. + dataOrStream = data; + } + } + if (data) { + const blob = new Blob([data]); + try { + thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); + } catch (e) { + log.error( + `Failed to generate ${fileTypeInfo.exactType} thumbnail`, + e, + ); + } + } + } + + if (!thumbnail) { + thumbnail = fallbackThumbnail(); + hasStaticThumbnail = true; + } + + return { + filedata: dataOrStream, + thumbnail, + hasStaticThumbnail, + }; +}; const _readFile = async (file: File, fileTypeInfo: FileTypeInfo) => { const dataOrStream = diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index c37644b700..03ef369826 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -646,7 +646,11 @@ async function downloadFileDesktop( fs.exists, ); const imageStream = generateStreamFromArrayBuffer(imageData); - await writeStream(`${downloadDir}/${imageExportName}`, imageStream); + await writeStream( + electron, + `${downloadDir}/${imageExportName}`, + imageStream, + ); try { const videoExportName = await safeFileName( downloadDir, @@ -654,7 +658,11 @@ async function downloadFileDesktop( fs.exists, ); const videoStream = generateStreamFromArrayBuffer(videoData); - await writeStream(`${downloadDir}/${videoExportName}`, videoStream); + await writeStream( + electron, + `${downloadDir}/${videoExportName}`, + videoStream, + ); } catch (e) { await fs.rm(`${downloadDir}/${imageExportName}`); throw e; @@ -665,7 +673,11 @@ async function downloadFileDesktop( file.metadata.title, fs.exists, ); - await writeStream(`${downloadDir}/${fileExportName}`, updatedStream); + await writeStream( + electron, + `${downloadDir}/${fileExportName}`, + updatedStream, + ); } } diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index 52da84f99d..a9a76b41be 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -4,13 +4,18 @@ * NOTE: These functions only work when we're running in our desktop app. */ +import type { Electron } from "@/next/types/ipc"; + /** * Stream the given file from the user's local filesystem. * - * **This only works when we're running in our desktop app**. It uses the + * This only works when we're running in our desktop app since it uses the * "stream://" protocol handler exposed by our custom code in the Node.js layer. * See: [Note: IPC streams]. * + * To avoid accidentally invoking it in a non-desktop app context, it requires + * the {@link Electron} object as a parameter (even though it doesn't use it). + * * @param path The path on the file on the user's local filesystem whose * contents we want to stream. * @@ -23,6 +28,7 @@ * * The size is the size of the file that we'll be reading from disk. */ export const readStream = async ( + _: Electron, path: string, ): Promise<{ response: Response; size: number }> => { const req = new Request(`stream://read${path}`, { @@ -47,14 +53,22 @@ export const readStream = async ( /** * Write the given stream to a file on the local machine. * - * **This only works when we're running in our desktop app**. It uses the + * This only works when we're running in our desktop app since it uses the * "stream://" protocol handler exposed by our custom code in the Node.js layer. * See: [Note: IPC streams]. * + * To avoid accidentally invoking it in a non-desktop app context, it requires + * the {@link Electron} object as a parameter (even though it doesn't use it). + * * @param path The path on the local machine where to write the file to. + * * @param stream The stream which should be written into the file. - * */ -export const writeStream = async (path: string, stream: ReadableStream) => { + */ +export const writeStream = async ( + _: Electron, + path: string, + stream: ReadableStream, +) => { // TODO(MR): This doesn't currently work. // // Not sure what I'm doing wrong here; I've opened an issue upstream From 308d8179b04e0f2df7b978c09da850233d2b872e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 14:49:53 +0530 Subject: [PATCH 28/82] Rework --- .../src/services/upload/uploadService.ts | 121 +++++++++--------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 6ec2d370f1..94f8661d4d 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -374,7 +374,7 @@ class ModuleState { * * Note the double negative when it is used. */ - isNativeImageThumbnailCreationNotAvailable = false; + isNativeImageThumbnailGenerationNotAvailable = false; } const moduleState = new ModuleState(); @@ -431,12 +431,11 @@ const readFileOrPath = async ( fileOrPath: File | string, fileTypeInfo: FileTypeInfo, ): Promise => { - let file: File | undefined; let dataOrStream: Uint8Array | DataStream; let fileSize: number; if (fileOrPath instanceof File) { - file = fileOrPath; + const file = fileOrPath; fileSize = file.size; dataOrStream = fileSize > MULTIPART_PART_SIZE @@ -458,80 +457,82 @@ const readFileOrPath = async ( let hasStaticThumbnail = false; const electron = globalThis.electron; - if (electron) { - // On Windows native thumbnail creation for images is not yet implemented. - const notAvailable = - fileTypeInfo.fileType == FILE_TYPE.IMAGE && - moduleState.isNativeImageThumbnailCreationNotAvailable; + const notAvailable = + fileTypeInfo.fileType == FILE_TYPE.IMAGE && + moduleState.isNativeImageThumbnailGenerationNotAvailable; + // 1. Native thumbnail generation. + if (electron && !notAvailable) { try { - if (!notAvailable) { - if (fileOrPath instanceof File) { - if (dataOrStream instanceof Uint8Array) { - thumbnail = await generateThumbnailNative( - electron, - dataOrStream, - fileTypeInfo, - ); - } - } else { + if (fileOrPath instanceof File) { + if (dataOrStream instanceof Uint8Array) { thumbnail = await generateThumbnailNative( electron, - fileOrPath, + dataOrStream, fileTypeInfo, ); + } else { + // This was large enough to need streaming, and trying to + // read it into memory or copying over IPC might cause us to + // run out of memory. So skip the native generation for it, + // instead let it get processed by the browser based + // thumbnailer (case 2). } + } else { + thumbnail = await generateThumbnailNative( + electron, + fileOrPath, + fileTypeInfo, + ); } } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeImageThumbnailCreationNotAvailable = true; + moduleState.isNativeImageThumbnailGenerationNotAvailable = true; } else { - log.error("Native thumbnail creation failed", e); + log.error("Native thumbnail generation failed", e); } } } - // If needed, fallback to browser based thumbnail generation. First, see if - // we already have a file (which is also a blob). - if (!thumbnail && file) { - try { - thumbnail = await generateThumbnailWeb(file, fileTypeInfo); - } catch (e) { - log.error( - `Failed to generate ${fileTypeInfo.exactType} thumbnail`, - e, - ); - } - } - - // Otherwise see if the data is small enough to read in memory. if (!thumbnail) { - let data: Uint8Array | undefined; - if (dataOrStream instanceof Uint8Array) { - data = dataOrStream; + let blob: Blob | undefined; + if (fileOrPath instanceof File) { + // 2. Browser based thumbnail generation for `File`s. + blob = fileOrPath; } else { - // Read the stream into memory, since the our web based thumbnail - // generation methods need the entire file in memory. Don't try this - // fallback for huge files though lest we run out of memory. - if (fileSize < 100 * 1024 * 1024 /* 100 MB */) { - data = new Uint8Array( - await new Response(dataOrStream.stream).arrayBuffer(), - ); - // The Readable stream cannot be read twice, so also overwrite - // the stream with the data we read. - dataOrStream = data; + // 3. Browser based thumbnail generation for paths. + if (dataOrStream instanceof Uint8Array) { + blob = new Blob([dataOrStream]); + } else { + // Read the stream into memory. Don't try this fallback for huge + // files though lest we run out of memory. + if (fileSize < 100 * 1024 * 1024 /* 100 MB */) { + const data = new Uint8Array( + await new Response(dataOrStream.stream).arrayBuffer(), + ); + // The Readable stream cannot be read twice, so also + // overwrite the stream with the data we read. + dataOrStream = data; + blob = new Blob([data]); + } else { + // There isn't a normal scenario where this should happen. + // Case 1, should've already worked, and the only known + // reason it'd have been skipped is for image files on + // Windows, but those should be less than 100 MB. + // + // So don't risk running out of memory for a case we don't + // comprehend. + log.error( + `Not using browser based thumbnail generation fallback for large file at path ${fileOrPath}`, + ); + } } } - if (data) { - const blob = new Blob([data]); - try { - thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); - } catch (e) { - log.error( - `Failed to generate ${fileTypeInfo.exactType} thumbnail`, - e, - ); - } + + try { + thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); + } catch (e) { + log.error("Web thumbnail creation failed", e); } } @@ -590,7 +591,7 @@ const _readPath = async (path: string, fileTypeInfo: FileTypeInfo) => { // On Windows native thumbnail creation for images is not yet implemented. const notAvailable = fileTypeInfo.fileType == FILE_TYPE.IMAGE && - moduleState.isNativeImageThumbnailCreationNotAvailable; + moduleState.isNativeImageThumbnailGenerationNotAvailable; try { if (!notAvailable) { @@ -602,7 +603,7 @@ const _readPath = async (path: string, fileTypeInfo: FileTypeInfo) => { } } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeImageThumbnailCreationNotAvailable = true; + moduleState.isNativeImageThumbnailGenerationNotAvailable = true; } else { log.error("Native thumbnail creation failed", e); } From 7f9563ab9aa52d7c0d370fd17c2d1a93fd12d0bd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:02:43 +0530 Subject: [PATCH 29/82] Possible approach --- .../src/services/upload/uploadService.ts | 176 ++++-------------- 1 file changed, 33 insertions(+), 143 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 94f8661d4d..982ae78ca2 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,10 +1,6 @@ import { encodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; -import { - basename, - convertBytesToHumanReadable, - getFileNameSize, -} from "@/next/file"; +import { basename, convertBytesToHumanReadable } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage } from "@/next/types/ipc"; @@ -358,8 +354,8 @@ const readAsset = async ( { isLivePhoto, file, livePhotoAssets }: UploadAsset, ) => { return isLivePhoto - ? await readLivePhoto(fileTypeInfo, livePhotoAssets) - : await readFile(fileTypeInfo, file); + ? await readLivePhoto(livePhotoAssets, fileTypeInfo) + : await readImageOrVideo(file, fileTypeInfo); }; // TODO(MR): Merge with the uploader @@ -429,8 +425,7 @@ const moduleState = new ModuleState(); */ const readFileOrPath = async ( fileOrPath: File | string, - fileTypeInfo: FileTypeInfo, -): Promise => { +): Promise<{ dataOrStream: Uint8Array | DataStream; fileSize: number }> => { let dataOrStream: Uint8Array | DataStream; let fileSize: number; @@ -453,6 +448,24 @@ const readFileOrPath = async ( } } + return { dataOrStream, fileSize }; +}; + +/** + * Augment the given {@link dataOrStream} with thumbnail information. + * + * This is a companion method for {@link readFileOrPath}, and can be used to + * convert the result of {@link readFileOrPath} into an {@link FileInMemory}. + * + * Note: The returned dataOrStream might be different from the one that we + * provide to it. + */ +const withThumbnail = async ( + fileOrPath: File | string, + fileTypeInfo: FileTypeInfo, + dataOrStream: Uint8Array | DataStream, + fileSize: number, +): Promise => { let thumbnail: Uint8Array | undefined; let hasStaticThumbnail = false; @@ -548,141 +561,18 @@ const readFileOrPath = async ( }; }; -const _readFile = async (file: File, fileTypeInfo: FileTypeInfo) => { - const dataOrStream = - file.size > MULTIPART_PART_SIZE - ? getFileStream(file, FILE_READER_CHUNK_SIZE) - : new Uint8Array(await file.arrayBuffer()); - - let thumbnail: Uint8Array; - let hasStaticThumbnail = false; - - try { - thumbnail = await generateThumbnailWeb(file, fileTypeInfo); - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - thumbnail = fallbackThumbnail(); - hasStaticThumbnail = true; - } - - return { - filedata: dataOrStream, - thumbnail, - hasStaticThumbnail, - }; -}; - -const _readPath = async (path: string, fileTypeInfo: FileTypeInfo) => { - const electron = ensureElectron(); - - let dataOrStream: Uint8Array | DataStream; - - const { response, size } = await readStream(path); - if (size > MULTIPART_PART_SIZE) { - const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); - dataOrStream = { stream: response.body, chunkCount }; - } else { - dataOrStream = new Uint8Array(await response.arrayBuffer()); - } - - let thumbnail: Uint8Array | undefined; - let hasStaticThumbnail = false; - - // On Windows native thumbnail creation for images is not yet implemented. - const notAvailable = - fileTypeInfo.fileType == FILE_TYPE.IMAGE && - moduleState.isNativeImageThumbnailGenerationNotAvailable; - - try { - if (!notAvailable) { - thumbnail = await generateThumbnailNative( - electron, - path, - fileTypeInfo, - ); - } - } catch (e) { - if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeImageThumbnailGenerationNotAvailable = true; - } else { - log.error("Native thumbnail creation failed", e); - } - } - - // If needed, fallback to browser based thumbnail generation. - if (!thumbnail) { - let data: Uint8Array | undefined; - if (dataOrStream instanceof Uint8Array) { - data = dataOrStream; - } else { - // Read the stream into memory, since the our web based thumbnail - // generation methods need the entire file in memory. Don't try this - // fallback for huge files though lest we run out of memory. - if (size < 100 * 1024 * 1024 /* 100 MB */) { - data = new Uint8Array( - await new Response(dataOrStream.stream).arrayBuffer(), - ); - // The Readable stream cannot be read twice, so also overwrite - // the stream with the data we read. - dataOrStream = data; - } - } - if (data) { - const blob = new Blob([data]); - try { - thumbnail = await generateThumbnailWeb(blob, fileTypeInfo); - } catch (e) { - log.error( - `Failed to generate ${fileTypeInfo.exactType} thumbnail`, - e, - ); - } - } - } - - if (!thumbnail) { - thumbnail = fallbackThumbnail(); - hasStaticThumbnail = true; - } - - return { - filedata: dataOrStream, - thumbnail, - hasStaticThumbnail, - }; - - // const dataOrPath = - // fileOrPath instanceof File - // ? new Uint8Array(await fileOrPath.arrayBuffer()) - // : fileOrPath; - - // let thumbnail: Uint8Array; - - // try { - // const thumbnail = - // fileTypeInfo.fileType === FILE_TYPE.IMAGE - // ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - // : await generateVideoThumbnail(blob); - - // if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - // return { thumbnail, hasStaticThumbnail: false }; - // } catch (e) { - // log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - // return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - // } - - // if (filedata instanceof Uint8Array) { - // } else { - // filedata.stream; - // } - - log.info(`read file data successfully ${getFileNameSize(rawFile)} `); -}; - -async function readLivePhoto( +const readImageOrVideo = async ( + fileOrPath: File | string, fileTypeInfo: FileTypeInfo, +) => { + const { dataOrStream, fileSize } = await readFileOrPath(fileOrPath); + return withThumbnail(fileOrPath, fileTypeInfo, dataOrStream, fileSize); +}; + +const readLivePhoto = async ( livePhotoAssets: LivePhotoAssets, -) { + fileTypeInfo: FileTypeInfo, +) => { const imageData = await getUint8ArrayView(livePhotoAssets.image); const videoData = await getUint8ArrayView(livePhotoAssets.video); @@ -706,7 +596,7 @@ async function readLivePhoto( thumbnail, hasStaticThumbnail, }; -} +}; export async function extractFileMetadata( worker: Remote, From be2d8c45d0f9b2a17134ad86aba6a3523536d4f5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:26:56 +0530 Subject: [PATCH 30/82] ReadLivePhoto --- .../src/services/upload/uploadService.ts | 59 +++++++++++++------ web/apps/photos/src/types/upload/index.ts | 6 +- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 982ae78ca2..b49b2937e8 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -44,7 +44,7 @@ import { UploadURL, isDataStream, type FileWithCollection2, - type LivePhotoAssets, + type LivePhotoAssets2, type UploadAsset2, } from "types/upload"; import { @@ -53,7 +53,7 @@ import { } from "utils/magicMetadata"; import { readStream } from "utils/native-stream"; import { findMatchingExistingFiles } from "utils/upload"; -import { getFileStream, getUint8ArrayView } from "../readerService"; +import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, @@ -351,7 +351,7 @@ export const getFileName = (file: File | ElectronFile | string) => const readAsset = async ( fileTypeInfo: FileTypeInfo, - { isLivePhoto, file, livePhotoAssets }: UploadAsset, + { isLivePhoto, file, livePhotoAssets }: UploadAsset2, ) => { return isLivePhoto ? await readLivePhoto(livePhotoAssets, fileTypeInfo) @@ -520,9 +520,7 @@ const withThumbnail = async ( // Read the stream into memory. Don't try this fallback for huge // files though lest we run out of memory. if (fileSize < 100 * 1024 * 1024 /* 100 MB */) { - const data = new Uint8Array( - await new Response(dataOrStream.stream).arrayBuffer(), - ); + const data = await readEntireStream(dataOrStream.stream); // The Readable stream cannot be read twice, so also // overwrite the stream with the data we read. dataOrStream = data; @@ -561,6 +559,17 @@ const withThumbnail = async ( }; }; +/** + * Read the entirety of a readable stream. + * + * It is not recommended to use this for large (say, multi-hundred MB) files. It + * is provided as a syntactic shortcut for cases where we already know that the + * size of the stream will be reasonable enough to be read in its entirety + * without us running out of memory. + */ +const readEntireStream = async (stream: ReadableStream) => + new Uint8Array(await new Response(stream).arrayBuffer()); + const readImageOrVideo = async ( fileOrPath: File | string, fileTypeInfo: FileTypeInfo, @@ -570,28 +579,42 @@ const readImageOrVideo = async ( }; const readLivePhoto = async ( - livePhotoAssets: LivePhotoAssets, + livePhotoAssets: LivePhotoAssets2, fileTypeInfo: FileTypeInfo, ) => { - const imageData = await getUint8ArrayView(livePhotoAssets.image); - - const videoData = await getUint8ArrayView(livePhotoAssets.video); - - const imageBlob = new Blob([imageData]); - const { thumbnail, hasStaticThumbnail } = await generateThumbnail( - imageBlob, + const readImage = await readFileOrPath(livePhotoAssets.image); + const { + filedata: imageDataOrStream, + thumbnail, + hasStaticThumbnail, + } = await withThumbnail( + livePhotoAssets.image, { exactType: fileTypeInfo.imageType, fileType: FILE_TYPE.IMAGE, }, + readImage.dataOrStream, + readImage.fileSize, ); + const readVideo = await readFileOrPath(livePhotoAssets.video); + + // We can revisit this later, but the existing code always read the + // full files into memory here, and to avoid changing the rest of + // the scaffolding retain the same behaviour. + // + // This is a reasonable assumption too, since the videos + // corresponding to live photos are only a couple of seconds long. + const toData = async (dataOrStream: Uint8Array | DataStream) => + dataOrStream instanceof Uint8Array + ? dataOrStream + : await readEntireStream(dataOrStream.stream); return { filedata: await encodeLivePhoto({ - imageFileName: livePhotoAssets.image.name, - imageData, - videoFileName: livePhotoAssets.video.name, - videoData, + imageFileName: getFileName(livePhotoAssets.image), + imageData: await toData(imageDataOrStream), + videoFileName: getFileName(livePhotoAssets.video), + videoData: await toData(readVideo.dataOrStream), }), thumbnail, hasStaticThumbnail, diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index f9d744dee7..00bf7c8b86 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -90,13 +90,13 @@ export interface FileWithCollection extends UploadAsset { export interface UploadAsset2 { isLivePhoto?: boolean; - file?: File | ElectronFile | string; + file?: File | string; livePhotoAssets?: LivePhotoAssets2; } export interface LivePhotoAssets2 { - image: File | ElectronFile | string; - video: File | ElectronFile | string; + image: File | string; + video: File | string; } export interface FileWithCollection2 extends UploadAsset2 { From 91afe6811188d5e5e17888f66eec7e58f68a0553 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:39:06 +0530 Subject: [PATCH 31/82] Cluster --- .../src/services/upload/metadataService.ts | 59 ++++++++++++++++- .../src/services/upload/uploadService.ts | 64 +++---------------- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index 367d79bbad..aa0b474b22 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -29,6 +29,7 @@ import { type FileWithCollection, type FileWithCollection2, type LivePhotoAssets2, + type UploadAsset2, } from "types/upload"; import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; @@ -349,7 +350,63 @@ export async function getLivePhotoFileType( }; } -export async function extractLivePhotoMetadata( +export const extractAssetMetadata = async ( + worker: Remote, + parsedMetadataJSONMap: ParsedMetadataJSONMap, + { isLivePhoto, file, livePhotoAssets }: UploadAsset2, + collectionID: number, + fileTypeInfo: FileTypeInfo, +): Promise => { + return isLivePhoto + ? await extractLivePhotoMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + livePhotoAssets, + ) + : await extractFileMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + file, + ); +}; + +async function extractFileMetadata( + worker: Remote, + parsedMetadataJSONMap: ParsedMetadataJSONMap, + collectionID: number, + fileTypeInfo: FileTypeInfo, + rawFile: File | ElectronFile | string, +): Promise { + const rawFileName = getFileName(rawFile); + let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); + let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); + + if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { + key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); + googleMetadata = parsedMetadataJSONMap.get(key); + } + + const { metadata, publicMagicMetadata } = await extractMetadata( + worker, + /* TODO(MR): ElectronFile changes */ + rawFile as File | ElectronFile, + fileTypeInfo, + ); + + for (const [key, value] of Object.entries(googleMetadata ?? {})) { + if (!value) { + continue; + } + metadata[key] = value; + } + return { metadata, publicMagicMetadata }; +} + +async function extractLivePhotoMetadata( worker: Remote, parsedMetadataJSONMap: ParsedMetadataJSONMap, collectionID: number, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b49b2937e8..2bed3d9334 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -56,14 +56,10 @@ import { findMatchingExistingFiles } from "utils/upload"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { - MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, - extractLivePhotoMetadata, - extractMetadata, - getClippedMetadataJSONMapKeyForFile, + extractAssetMetadata, getLivePhotoFileType, getLivePhotoName, getLivePhotoSize, - getMetadataJSONMapKeyForFile, } from "./metadataService"; import { uploadStreamUsingMultipart } from "./multiPartUploadService"; import publicUploadHttpClient from "./publicUploadHttpClient"; @@ -145,25 +141,17 @@ class UploadService { async extractAssetMetadata( worker: Remote, - { isLivePhoto, file, livePhotoAssets }: UploadAsset2, + uploadAsset: UploadAsset2, collectionID: number, fileTypeInfo: FileTypeInfo, ): Promise { - return isLivePhoto - ? extractLivePhotoMetadata( - worker, - this.parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - livePhotoAssets, - ) - : await extractFileMetadata( - worker, - this.parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - file, - ); + return await extractAssetMetadata( + worker, + this.parsedMetadataJSONMap, + uploadAsset, + collectionID, + fileTypeInfo, + ); } async encryptAsset( @@ -621,38 +609,6 @@ const readLivePhoto = async ( }; }; -export async function extractFileMetadata( - worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, - collectionID: number, - fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile | string, -): Promise { - const rawFileName = getFileName(rawFile); - let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); - let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); - - if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { - key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); - googleMetadata = parsedMetadataJSONMap.get(key); - } - - const { metadata, publicMagicMetadata } = await extractMetadata( - worker, - /* TODO(MR): ElectronFile changes */ - rawFile as File | ElectronFile, - fileTypeInfo, - ); - - for (const [key, value] of Object.entries(googleMetadata ?? {})) { - if (!value) { - continue; - } - metadata[key] = value; - } - return { metadata, publicMagicMetadata }; -} - async function encryptFile( worker: Remote, file: FileWithMetadata, @@ -843,7 +799,7 @@ export async function uploader( } log.info(`reading asset ${fileNameSize}`); - const file = readAsset(fileTypeInfo, uploadAsset); + const file = await readAsset(fileTypeInfo, uploadAsset); if (file.hasStaticThumbnail) { metadata.hasStaticThumbnail = true; From 3a93a7a95684d7966ac575f5c6baca88fb884349 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:48:52 +0530 Subject: [PATCH 32/82] Prune --- .../src/services/upload/metadataService.ts | 5 +---- .../src/services/upload/uploadManager.ts | 15 ++++++++----- .../src/services/upload/uploadService.ts | 22 ++++++++++++------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index aa0b474b22..3944e6fa51 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -435,7 +435,7 @@ async function extractLivePhotoMetadata( return { metadata: { ...imageMetadata, - title: getLivePhotoName(livePhotoAssets), + title: getFileName(livePhotoAssets.image), fileType: FILE_TYPE.LIVE_PHOTO, imageHash: imageMetadata.hash, videoHash: videoHash, @@ -449,9 +449,6 @@ export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { return livePhotoAssets.image.size + livePhotoAssets.video.size; } -export const getLivePhotoName = ({ image }: LivePhotoAssets2) => - typeof image == "string" ? basename(image) : image.name; - export async function clusterLivePhotoFiles(mediaFiles: FileWithCollection2[]) { try { const analysedMediaFiles: FileWithCollection2[] = []; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 65ce7d77f2..68c1589f69 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -40,7 +40,12 @@ import { } from "./metadataService"; import { default as UIService, default as uiService } from "./uiService"; import uploadCancelService from "./uploadCancelService"; -import UploadService, { getFileName, uploader } from "./uploadService"; +import UploadService, { + assetName, + getAssetName, + getFileName, + uploader, +} from "./uploadService"; const MAX_CONCURRENT_UPLOADS = 4; @@ -132,7 +137,7 @@ class UploadManager { new Map( filesWithCollectionToUploadIn.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + getAssetName(mediaFile), ]), ), ); @@ -164,7 +169,7 @@ class UploadManager { new Map( analysedMediaFiles.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + assetName(mediaFile), ]), ), ); @@ -223,7 +228,7 @@ class UploadManager { new Map( filesWithCollectionToUploadIn.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + assetName(mediaFile), ]), ), ); @@ -255,7 +260,7 @@ class UploadManager { new Map( analysedMediaFiles.map((mediaFile) => [ mediaFile.localID, - UploadService.getAssetName(mediaFile), + assetName(mediaFile), ]), ), ); diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 2bed3d9334..13deeb51a8 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -58,7 +58,6 @@ import { getFileType } from "../typeDetectionService"; import { extractAssetMetadata, getLivePhotoFileType, - getLivePhotoName, getLivePhotoSize, } from "./metadataService"; import { uploadStreamUsingMultipart } from "./multiPartUploadService"; @@ -127,12 +126,6 @@ class UploadService { : getFileSize(file); } - getAssetName({ isLivePhoto, file, livePhotoAssets }: UploadAsset2) { - return isLivePhoto - ? getLivePhotoName(livePhotoAssets) - : getFileName(file); - } - getAssetFileType({ isLivePhoto, file, livePhotoAssets }: UploadAsset) { return isLivePhoto ? getLivePhotoFileType(livePhotoAssets) @@ -333,6 +326,19 @@ const constructPublicMagicMetadata = async ( function getFileSize(file: File | ElectronFile) { return file.size; } +export const getAssetName = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset) => + isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file); + +export const assetName = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset2) => + isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file); export const getFileName = (file: File | ElectronFile | string) => typeof file == "string" ? basename(file) : file.name; @@ -719,7 +725,7 @@ export async function uploader( const { collection, localID, ...uploadAsset2 } = fileWithCollection; /* TODO(MR): ElectronFile changes */ const uploadAsset = uploadAsset2 as UploadAsset; - const fileNameSize = `${uploadService.getAssetName( + const fileNameSize = `${assetName( fileWithCollection, )}_${convertBytesToHumanReadable(uploadService.getAssetSize(uploadAsset))}`; From b80f567e74e7477fbd92161461345d19ced0bf72 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:50:15 +0530 Subject: [PATCH 33/82] Rearrange --- .../src/services/upload/uploadService.ts | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 13deeb51a8..a0deffa59d 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -22,8 +22,8 @@ import { addToCollection } from "services/collectionService"; import { Collection } from "types/collection"; import { EnteFile, - FilePublicMagicMetadata, - FilePublicMagicMetadataProps, + type FilePublicMagicMetadata, + type FilePublicMagicMetadataProps, } from "types/file"; import { EncryptedMagicMetadata } from "types/magicMetadata"; import { @@ -120,18 +120,6 @@ class UploadService { this.pendingUploadCount--; } - getAssetSize({ isLivePhoto, file, livePhotoAssets }: UploadAsset) { - return isLivePhoto - ? getLivePhotoSize(livePhotoAssets) - : getFileSize(file); - } - - getAssetFileType({ isLivePhoto, file, livePhotoAssets }: UploadAsset) { - return isLivePhoto - ? getLivePhotoFileType(livePhotoAssets) - : getFileType(file); - } - async extractAssetMetadata( worker: Remote, uploadAsset: UploadAsset2, @@ -310,18 +298,8 @@ const uploadService = new UploadService(); export default uploadService; -const constructPublicMagicMetadata = async ( - publicMagicMetadataProps: FilePublicMagicMetadataProps, -): Promise => { - const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps( - publicMagicMetadataProps, - ); - - if (Object.values(nonEmptyPublicMagicMetadataProps)?.length === 0) { - return null; - } - return await updateMagicMetadata(publicMagicMetadataProps); -}; +export const getFileName = (file: File | ElectronFile | string) => + typeof file == "string" ? basename(file) : file.name; function getFileSize(file: File | ElectronFile) { return file.size; @@ -340,9 +318,19 @@ export const assetName = ({ }: UploadAsset2) => isLivePhoto ? getFileName(livePhotoAssets.image) : getFileName(file); -export const getFileName = (file: File | ElectronFile | string) => - typeof file == "string" ? basename(file) : file.name; +const getAssetSize = ({ isLivePhoto, file, livePhotoAssets }: UploadAsset) => { + return isLivePhoto ? getLivePhotoSize(livePhotoAssets) : getFileSize(file); +}; +const getAssetFileType = ({ + isLivePhoto, + file, + livePhotoAssets, +}: UploadAsset) => { + return isLivePhoto + ? getLivePhotoFileType(livePhotoAssets) + : getFileType(file); +}; const readAsset = async ( fileTypeInfo: FileTypeInfo, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, @@ -615,6 +603,19 @@ const readLivePhoto = async ( }; }; +const constructPublicMagicMetadata = async ( + publicMagicMetadataProps: FilePublicMagicMetadataProps, +): Promise => { + const nonEmptyPublicMagicMetadataProps = getNonEmptyMagicMetadataProps( + publicMagicMetadataProps, + ); + + if (Object.values(nonEmptyPublicMagicMetadataProps)?.length === 0) { + return null; + } + return await updateMagicMetadata(publicMagicMetadataProps); +}; + async function encryptFile( worker: Remote, file: FileWithMetadata, @@ -727,7 +728,7 @@ export async function uploader( const uploadAsset = uploadAsset2 as UploadAsset; const fileNameSize = `${assetName( fileWithCollection, - )}_${convertBytesToHumanReadable(uploadService.getAssetSize(uploadAsset))}`; + )}_${convertBytesToHumanReadable(getAssetSize(uploadAsset))}`; log.info(`uploader called for ${fileNameSize}`); UIService.setFileProgress(localID, 0); @@ -737,12 +738,13 @@ export async function uploader( try { const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB - fileSize = uploadService.getAssetSize(uploadAsset); + fileSize = getAssetSize(uploadAsset); if (fileSize >= maxFileSize) { return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; } log.info(`getting filetype for ${fileNameSize}`); - fileTypeInfo = await uploadService.getAssetFileType(uploadAsset); + fileTypeInfo = await getAssetFileType(uploadAsset); + log.info( `got filetype for ${fileNameSize} - ${JSON.stringify(fileTypeInfo)}`, ); From 14427b60116e7fc4bf8422f3d6dba6533c39f4cb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 15:57:09 +0530 Subject: [PATCH 34/82] Remove unused --- web/apps/photos/src/utils/upload/index.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index ac05122aa9..925d19b452 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -12,7 +12,6 @@ import { } from "types/upload"; const TYPE_JSON = "json"; -const DEDUPE_COLLECTION = new Set(["icloud library", "icloudlibrary"]); export function findMatchingExistingFiles( existingFiles: EnteFile[], @@ -27,11 +26,6 @@ export function findMatchingExistingFiles( return matchingFiles; } -export function shouldDedupeAcrossCollection(collectionName: string): boolean { - // using set to avoid unnecessary regex for removing spaces for each upload - return DEDUPE_COLLECTION.has(collectionName.toLocaleLowerCase()); -} - export function areFilesSame( existingFile: Metadata, newFile: Metadata, From c7e0986b1210bcff1178109b5c007b65a3fc024a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 16:01:22 +0530 Subject: [PATCH 35/82] Inline and reorder --- .../src/services/upload/uploadService.ts | 417 ++++++++++-------- web/apps/photos/src/utils/upload/index.ts | 62 --- 2 files changed, 240 insertions(+), 239 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index a0deffa59d..fa60476051 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -45,6 +45,7 @@ import { isDataStream, type FileWithCollection2, type LivePhotoAssets2, + type Metadata, type UploadAsset2, } from "types/upload"; import { @@ -52,7 +53,7 @@ import { updateMagicMetadata, } from "utils/magicMetadata"; import { readStream } from "utils/native-stream"; -import { findMatchingExistingFiles } from "utils/upload"; +import { findMatchingExistingFiles, hasFileHash } from "utils/upload"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { @@ -298,6 +299,190 @@ const uploadService = new UploadService(); export default uploadService; +interface UploadResponse { + fileUploadResult: UPLOAD_RESULT; + uploadedFile?: EnteFile; +} + +export async function uploader( + worker: Remote, + existingFiles: EnteFile[], + fileWithCollection: FileWithCollection2, + uploaderName: string, +): Promise { + const { collection, localID, ...uploadAsset2 } = fileWithCollection; + /* TODO(MR): ElectronFile changes */ + const uploadAsset = uploadAsset2 as UploadAsset; + const fileNameSize = `${assetName( + fileWithCollection, + )}_${convertBytesToHumanReadable(getAssetSize(uploadAsset))}`; + + log.info(`uploader called for ${fileNameSize}`); + UIService.setFileProgress(localID, 0); + await wait(0); + let fileTypeInfo: FileTypeInfo; + let fileSize: number; + try { + const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB + + fileSize = getAssetSize(uploadAsset); + if (fileSize >= maxFileSize) { + return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; + } + log.info(`getting filetype for ${fileNameSize}`); + fileTypeInfo = await getAssetFileType(uploadAsset); + + log.info( + `got filetype for ${fileNameSize} - ${JSON.stringify(fileTypeInfo)}`, + ); + + log.info(`extracting metadata ${fileNameSize}`); + const { metadata, publicMagicMetadata } = + await uploadService.extractAssetMetadata( + worker, + uploadAsset, + collection.id, + fileTypeInfo, + ); + + const matchingExistingFiles = findMatchingExistingFiles( + existingFiles, + metadata, + ); + log.debug( + () => + `matchedFileList: ${matchingExistingFiles + .map((f) => `${f.id}-${f.metadata.title}`) + .join(",")}`, + ); + if (matchingExistingFiles?.length) { + const matchingExistingFilesCollectionIDs = + matchingExistingFiles.map((e) => e.collectionID); + log.debug( + () => + `matched file collectionIDs:${matchingExistingFilesCollectionIDs} + and collectionID:${collection.id}`, + ); + if (matchingExistingFilesCollectionIDs.includes(collection.id)) { + log.info( + `file already present in the collection , skipped upload for ${fileNameSize}`, + ); + const sameCollectionMatchingExistingFile = + matchingExistingFiles.find( + (f) => f.collectionID === collection.id, + ); + return { + fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED, + uploadedFile: sameCollectionMatchingExistingFile, + }; + } else { + log.info( + `same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize} ,adding symlink`, + ); + // any of the matching file can used to add a symlink + const resultFile = Object.assign({}, matchingExistingFiles[0]); + resultFile.collectionID = collection.id; + await addToCollection(collection, [resultFile]); + return { + fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK, + uploadedFile: resultFile, + }; + } + } + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } + log.info(`reading asset ${fileNameSize}`); + + const file = await readAsset(fileTypeInfo, uploadAsset); + + if (file.hasStaticThumbnail) { + metadata.hasStaticThumbnail = true; + } + + const pubMagicMetadata = await constructPublicMagicMetadata({ + ...publicMagicMetadata, + uploaderName, + }); + + const fileWithMetadata: FileWithMetadata = { + localID, + filedata: file.filedata, + thumbnail: file.thumbnail, + metadata, + pubMagicMetadata, + }; + + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } + log.info(`encryptAsset ${fileNameSize}`); + const encryptedFile = await uploadService.encryptAsset( + worker, + fileWithMetadata, + collection.key, + ); + + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } + log.info(`uploadToBucket ${fileNameSize}`); + const logger: Logger = (message: string) => { + log.info(message, `fileNameSize: ${fileNameSize}`); + }; + const backupedFile: BackupedFile = await uploadService.uploadToBucket( + logger, + encryptedFile.file, + ); + + const uploadFile: UploadFile = uploadService.getUploadFile( + collection, + backupedFile, + encryptedFile.fileKey, + ); + log.info(`uploading file to server ${fileNameSize}`); + + const uploadedFile = await uploadService.uploadFile(uploadFile); + + log.info(`${fileNameSize} successfully uploaded`); + + return { + fileUploadResult: metadata.hasStaticThumbnail + ? UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL + : UPLOAD_RESULT.UPLOADED, + uploadedFile: uploadedFile, + }; + } catch (e) { + log.info(`upload failed for ${fileNameSize} ,error: ${e.message}`); + if ( + e.message !== CustomError.UPLOAD_CANCELLED && + e.message !== CustomError.UNSUPPORTED_FILE_FORMAT + ) { + log.error( + `file upload failed - ${JSON.stringify({ + fileFormat: fileTypeInfo?.exactType, + fileSize: convertBytesToHumanReadable(fileSize), + })}`, + e, + ); + } + const error = handleUploadError(e); + switch (error.message) { + case CustomError.ETAG_MISSING: + return { fileUploadResult: UPLOAD_RESULT.BLOCKED }; + case CustomError.UNSUPPORTED_FILE_FORMAT: + return { fileUploadResult: UPLOAD_RESULT.UNSUPPORTED }; + case CustomError.FILE_TOO_LARGE: + return { + fileUploadResult: + UPLOAD_RESULT.LARGER_THAN_AVAILABLE_STORAGE, + }; + default: + return { fileUploadResult: UPLOAD_RESULT.FAILED }; + } + } +} + export const getFileName = (file: File | ElectronFile | string) => typeof file == "string" ? basename(file) : file.name; @@ -712,186 +897,64 @@ async function encryptFileStream( }; } -interface UploadResponse { - fileUploadResult: UPLOAD_RESULT; - uploadedFile?: EnteFile; +export function findMatchingExistingFiles( + existingFiles: EnteFile[], + newFileMetadata: Metadata, +): EnteFile[] { + const matchingFiles: EnteFile[] = []; + for (const existingFile of existingFiles) { + if (areFilesSame(existingFile.metadata, newFileMetadata)) { + matchingFiles.push(existingFile); + } + } + return matchingFiles; } -export async function uploader( - worker: Remote, - existingFiles: EnteFile[], - fileWithCollection: FileWithCollection2, - uploaderName: string, -): Promise { - const { collection, localID, ...uploadAsset2 } = fileWithCollection; - /* TODO(MR): ElectronFile changes */ - const uploadAsset = uploadAsset2 as UploadAsset; - const fileNameSize = `${assetName( - fileWithCollection, - )}_${convertBytesToHumanReadable(getAssetSize(uploadAsset))}`; - - log.info(`uploader called for ${fileNameSize}`); - UIService.setFileProgress(localID, 0); - await wait(0); - let fileTypeInfo: FileTypeInfo; - let fileSize: number; - try { - const maxFileSize = 4 * 1024 * 1024 * 1024; // 4 GB - - fileSize = getAssetSize(uploadAsset); - if (fileSize >= maxFileSize) { - return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; - } - log.info(`getting filetype for ${fileNameSize}`); - fileTypeInfo = await getAssetFileType(uploadAsset); - - log.info( - `got filetype for ${fileNameSize} - ${JSON.stringify(fileTypeInfo)}`, - ); - - log.info(`extracting metadata ${fileNameSize}`); - const { metadata, publicMagicMetadata } = - await uploadService.extractAssetMetadata( - worker, - uploadAsset, - collection.id, - fileTypeInfo, - ); - - const matchingExistingFiles = findMatchingExistingFiles( - existingFiles, - metadata, - ); - log.debug( - () => - `matchedFileList: ${matchingExistingFiles - .map((f) => `${f.id}-${f.metadata.title}`) - .join(",")}`, - ); - if (matchingExistingFiles?.length) { - const matchingExistingFilesCollectionIDs = - matchingExistingFiles.map((e) => e.collectionID); - log.debug( - () => - `matched file collectionIDs:${matchingExistingFilesCollectionIDs} - and collectionID:${collection.id}`, - ); - if (matchingExistingFilesCollectionIDs.includes(collection.id)) { - log.info( - `file already present in the collection , skipped upload for ${fileNameSize}`, - ); - const sameCollectionMatchingExistingFile = - matchingExistingFiles.find( - (f) => f.collectionID === collection.id, - ); - return { - fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED, - uploadedFile: sameCollectionMatchingExistingFile, - }; - } else { - log.info( - `same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize} ,adding symlink`, - ); - // any of the matching file can used to add a symlink - const resultFile = Object.assign({}, matchingExistingFiles[0]); - resultFile.collectionID = collection.id; - await addToCollection(collection, [resultFile]); - return { - fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK, - uploadedFile: resultFile, - }; - } - } - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`reading asset ${fileNameSize}`); - - const file = await readAsset(fileTypeInfo, uploadAsset); - - if (file.hasStaticThumbnail) { - metadata.hasStaticThumbnail = true; - } - - const pubMagicMetadata = await constructPublicMagicMetadata({ - ...publicMagicMetadata, - uploaderName, - }); - - const fileWithMetadata: FileWithMetadata = { - localID, - filedata: file.filedata, - thumbnail: file.thumbnail, - metadata, - pubMagicMetadata, - }; - - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`encryptAsset ${fileNameSize}`); - const encryptedFile = await uploadService.encryptAsset( - worker, - fileWithMetadata, - collection.key, - ); - - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - log.info(`uploadToBucket ${fileNameSize}`); - const logger: Logger = (message: string) => { - log.info(message, `fileNameSize: ${fileNameSize}`); - }; - const backupedFile: BackupedFile = await uploadService.uploadToBucket( - logger, - encryptedFile.file, - ); - - const uploadFile: UploadFile = uploadService.getUploadFile( - collection, - backupedFile, - encryptedFile.fileKey, - ); - log.info(`uploading file to server ${fileNameSize}`); - - const uploadedFile = await uploadService.uploadFile(uploadFile); - - log.info(`${fileNameSize} successfully uploaded`); - - return { - fileUploadResult: metadata.hasStaticThumbnail - ? UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL - : UPLOAD_RESULT.UPLOADED, - uploadedFile: uploadedFile, - }; - } catch (e) { - log.info(`upload failed for ${fileNameSize} ,error: ${e.message}`); +export function areFilesSame( + existingFile: Metadata, + newFile: Metadata, +): boolean { + if (hasFileHash(existingFile) && hasFileHash(newFile)) { + return areFilesWithFileHashSame(existingFile, newFile); + } else { + /* + * The maximum difference in the creation/modification times of two similar files is set to 1 second. + * This is because while uploading files in the web - browsers and users could have set reduced + * precision of file times to prevent timing attacks and fingerprinting. + * Context: https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision + */ + const oneSecond = 1e6; if ( - e.message !== CustomError.UPLOAD_CANCELLED && - e.message !== CustomError.UNSUPPORTED_FILE_FORMAT + existingFile.fileType === newFile.fileType && + Math.abs(existingFile.creationTime - newFile.creationTime) < + oneSecond && + Math.abs(existingFile.modificationTime - newFile.modificationTime) < + oneSecond && + existingFile.title === newFile.title ) { - log.error( - `file upload failed - ${JSON.stringify({ - fileFormat: fileTypeInfo?.exactType, - fileSize: convertBytesToHumanReadable(fileSize), - })}`, - e, - ); - } - const error = handleUploadError(e); - switch (error.message) { - case CustomError.ETAG_MISSING: - return { fileUploadResult: UPLOAD_RESULT.BLOCKED }; - case CustomError.UNSUPPORTED_FILE_FORMAT: - return { fileUploadResult: UPLOAD_RESULT.UNSUPPORTED }; - case CustomError.FILE_TOO_LARGE: - return { - fileUploadResult: - UPLOAD_RESULT.LARGER_THAN_AVAILABLE_STORAGE, - }; - default: - return { fileUploadResult: UPLOAD_RESULT.FAILED }; + return true; + } else { + return false; } } } + +function areFilesWithFileHashSame( + existingFile: Metadata, + newFile: Metadata, +): boolean { + if ( + existingFile.fileType !== newFile.fileType || + existingFile.title !== newFile.title + ) { + return false; + } + if (existingFile.fileType === FILE_TYPE.LIVE_PHOTO) { + return ( + existingFile.imageHash === newFile.imageHash && + existingFile.videoHash === newFile.videoHash + ); + } else { + return existingFile.hash === newFile.hash; + } +} diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 925d19b452..c75cc606bc 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -13,72 +13,10 @@ import { const TYPE_JSON = "json"; -export function findMatchingExistingFiles( - existingFiles: EnteFile[], - newFileMetadata: Metadata, -): EnteFile[] { - const matchingFiles: EnteFile[] = []; - for (const existingFile of existingFiles) { - if (areFilesSame(existingFile.metadata, newFileMetadata)) { - matchingFiles.push(existingFile); - } - } - return matchingFiles; -} - -export function areFilesSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if (hasFileHash(existingFile) && hasFileHash(newFile)) { - return areFilesWithFileHashSame(existingFile, newFile); - } else { - /* - * The maximum difference in the creation/modification times of two similar files is set to 1 second. - * This is because while uploading files in the web - browsers and users could have set reduced - * precision of file times to prevent timing attacks and fingerprinting. - * Context: https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision - */ - const oneSecond = 1e6; - if ( - existingFile.fileType === newFile.fileType && - Math.abs(existingFile.creationTime - newFile.creationTime) < - oneSecond && - Math.abs(existingFile.modificationTime - newFile.modificationTime) < - oneSecond && - existingFile.title === newFile.title - ) { - return true; - } else { - return false; - } - } -} - export function hasFileHash(file: Metadata) { return file.hash || (file.imageHash && file.videoHash); } -export function areFilesWithFileHashSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if ( - existingFile.fileType !== newFile.fileType || - existingFile.title !== newFile.title - ) { - return false; - } - if (existingFile.fileType === FILE_TYPE.LIVE_PHOTO) { - return ( - existingFile.imageHash === newFile.imageHash && - existingFile.videoHash === newFile.videoHash - ); - } else { - return existingFile.hash === newFile.hash; - } -} - export function segregateMetadataAndMediaFiles( filesWithCollectionToUpload: FileWithCollection[], ) { From 5e5d66c2a2233595253a440be83c09a366328b1d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 16:04:19 +0530 Subject: [PATCH 36/82] Inline --- .../src/services/upload/uploadService.ts | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index fa60476051..835bbe8f33 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -5,10 +5,7 @@ import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage } from "@/next/types/ipc"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; -import { - B64EncryptionResult, - EncryptionResult, -} from "@ente/shared/crypto/types"; +import { EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; import { wait } from "@ente/shared/utils"; import { Remote } from "comlink"; @@ -19,7 +16,6 @@ import { UPLOAD_RESULT, } from "constants/upload"; import { addToCollection } from "services/collectionService"; -import { Collection } from "types/collection"; import { EnteFile, type FilePublicMagicMetadata, @@ -221,21 +217,6 @@ class UploadService { } } - getUploadFile( - collection: Collection, - backupedFile: BackupedFile, - fileKey: B64EncryptionResult, - ): UploadFile { - const uploadFile: UploadFile = { - collectionID: collection.id, - encryptedKey: fileKey.encryptedData, - keyDecryptionNonce: fileKey.nonce, - ...backupedFile, - }; - uploadFile; - return uploadFile; - } - private async getUploadURL() { if (this.uploadURLs.length === 0 && this.pendingUploadCount) { await this.fetchUploadURLs(); @@ -435,11 +416,12 @@ export async function uploader( encryptedFile.file, ); - const uploadFile: UploadFile = uploadService.getUploadFile( - collection, - backupedFile, - encryptedFile.fileKey, - ); + const uploadFile: UploadFile = { + collectionID: collection.id, + encryptedKey: encryptedFile.fileKey.encryptedData, + keyDecryptionNonce: encryptedFile.fileKey.nonce, + ...backupedFile, + }; log.info(`uploading file to server ${fileNameSize}`); const uploadedFile = await uploadService.uploadFile(uploadFile); From c5ab1811fb6aef55d5db85d9ca60c67ffe2386da Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 16:06:53 +0530 Subject: [PATCH 37/82] Inline and skip --- .../photos/src/services/upload/uploadService.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 835bbe8f33..57c3f6651a 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -49,7 +49,7 @@ import { updateMagicMetadata, } from "utils/magicMetadata"; import { readStream } from "utils/native-stream"; -import { findMatchingExistingFiles, hasFileHash } from "utils/upload"; +import { hasFileHash } from "utils/upload"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { @@ -132,14 +132,6 @@ class UploadService { ); } - async encryptAsset( - worker: Remote, - file: FileWithMetadata, - encryptionKey: string, - ): Promise { - return encryptFile(worker, file, encryptionKey); - } - async uploadToBucket( logger: Logger, file: ProcessedFile, @@ -397,8 +389,8 @@ export async function uploader( if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); } - log.info(`encryptAsset ${fileNameSize}`); - const encryptedFile = await uploadService.encryptAsset( + log.info(`encryptFile ${fileNameSize}`); + const encryptedFile = await encryptFile( worker, fileWithMetadata, collection.key, From 2e222d94096ec736e87806a9d6d0aa8374171add Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 16:15:57 +0530 Subject: [PATCH 38/82] Remove derived state --- .../src/services/upload/uploadManager.ts | 9 +--- .../src/services/upload/uploadService.ts | 41 ++++--------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 68c1589f69..0b9b485971 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -150,10 +150,6 @@ class UploadManager { UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.parseMetadataJSONFiles(metadataJSONFiles); - - UploadService.setParsedMetadataJSONMap( - this.parsedMetadataJSONMap, - ); } if (mediaFiles.length) { log.info(`clusterLivePhotoFiles started`); @@ -241,10 +237,6 @@ class UploadManager { UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.parseMetadataJSONFiles(metadataJSONFiles); - - UploadService.setParsedMetadataJSONMap( - this.parsedMetadataJSONMap, - ); } if (mediaFiles.length) { log.info(`clusterLivePhotoFiles started`); @@ -384,6 +376,7 @@ class UploadManager { worker, this.existingFiles, fileWithCollection, + this.parsedMetadataJSONMap, this.uploaderName, ); diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 57c3f6651a..4a82c448bd 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -26,12 +26,10 @@ import { BackupedFile, DataStream, EncryptedFile, - ExtractMetadataResult, FileInMemory, FileTypeInfo, FileWithMetadata, Logger, - ParsedMetadataJSON, ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, @@ -71,10 +69,6 @@ import UploadHttpClient from "./uploadHttpClient"; /** Upload files to cloud storage */ class UploadService { private uploadURLs: UploadURL[] = []; - private parsedMetadataJSONMap: ParsedMetadataJSONMap = new Map< - string, - ParsedMetadataJSON - >(); private uploaderName: string; @@ -97,10 +91,6 @@ class UploadService { await this.preFetchUploadURLs(); } - setParsedMetadataJSONMap(parsedMetadataJSONMap: ParsedMetadataJSONMap) { - this.parsedMetadataJSONMap = parsedMetadataJSONMap; - } - setUploaderName(uploaderName: string) { this.uploaderName = uploaderName; } @@ -117,21 +107,6 @@ class UploadService { this.pendingUploadCount--; } - async extractAssetMetadata( - worker: Remote, - uploadAsset: UploadAsset2, - collectionID: number, - fileTypeInfo: FileTypeInfo, - ): Promise { - return await extractAssetMetadata( - worker, - this.parsedMetadataJSONMap, - uploadAsset, - collectionID, - fileTypeInfo, - ); - } - async uploadToBucket( logger: Logger, file: ProcessedFile, @@ -281,6 +256,7 @@ export async function uploader( worker: Remote, existingFiles: EnteFile[], fileWithCollection: FileWithCollection2, + parsedMetadataJSONMap: ParsedMetadataJSONMap, uploaderName: string, ): Promise { const { collection, localID, ...uploadAsset2 } = fileWithCollection; @@ -310,13 +286,13 @@ export async function uploader( ); log.info(`extracting metadata ${fileNameSize}`); - const { metadata, publicMagicMetadata } = - await uploadService.extractAssetMetadata( - worker, - uploadAsset, - collection.id, - fileTypeInfo, - ); + const { metadata, publicMagicMetadata } = await extractAssetMetadata( + worker, + parsedMetadataJSONMap, + uploadAsset, + collection.id, + fileTypeInfo, + ); const matchingExistingFiles = findMatchingExistingFiles( existingFiles, @@ -490,6 +466,7 @@ const getAssetFileType = ({ ? getLivePhotoFileType(livePhotoAssets) : getFileType(file); }; + const readAsset = async ( fileTypeInfo: FileTypeInfo, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, From 00c9d78ec9f83e9362ccf6c8b7f1339ec7524732 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 18:36:42 +0530 Subject: [PATCH 39/82] Inline --- .../services/upload/multiPartUploadService.ts | 132 ------------------ .../src/services/upload/uploadService.ts | 127 ++++++++++++++++- web/apps/photos/src/utils/upload/index.ts | 2 - 3 files changed, 126 insertions(+), 135 deletions(-) delete mode 100644 web/apps/photos/src/services/upload/multiPartUploadService.ts diff --git a/web/apps/photos/src/services/upload/multiPartUploadService.ts b/web/apps/photos/src/services/upload/multiPartUploadService.ts deleted file mode 100644 index 1b4442710f..0000000000 --- a/web/apps/photos/src/services/upload/multiPartUploadService.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { CustomError } from "@ente/shared/error"; -import { - FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, -} from "constants/upload"; -import { DataStream, Logger, MultipartUploadURLs } from "types/upload"; -import * as convert from "xml-js"; -import UIService from "./uiService"; -import uploadCancelService from "./uploadCancelService"; -import UploadHttpClient from "./uploadHttpClient"; -import uploadService from "./uploadService"; - -interface PartEtag { - PartNumber: number; - ETag: string; -} - -function calculatePartCount(chunkCount: number) { - const partCount = Math.ceil( - chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - ); - return partCount; -} -export async function uploadStreamUsingMultipart( - logger: Logger, - fileLocalID: number, - dataStream: DataStream, -) { - const uploadPartCount = calculatePartCount(dataStream.chunkCount); - logger(`fetching ${uploadPartCount} urls for multipart upload`); - const multipartUploadURLs = - await uploadService.fetchMultipartUploadURLs(uploadPartCount); - logger(`fetched ${uploadPartCount} urls for multipart upload`); - - const fileObjectKey = await uploadStreamInParts( - logger, - multipartUploadURLs, - dataStream.stream, - fileLocalID, - uploadPartCount, - ); - return fileObjectKey; -} - -export async function uploadStreamInParts( - logger: Logger, - multipartUploadURLs: MultipartUploadURLs, - dataStream: ReadableStream, - fileLocalID: number, - uploadPartCount: number, -) { - const streamReader = dataStream.getReader(); - const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); - const partEtags: PartEtag[] = []; - logger(`uploading file in chunks`); - for (const [ - index, - fileUploadURL, - ] of multipartUploadURLs.partURLs.entries()) { - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - const uploadChunk = await combineChunksToFormUploadPart(streamReader); - const progressTracker = UIService.trackUploadProgress( - fileLocalID, - percentPerPart, - index, - ); - let eTag = null; - if (!uploadService.getIsCFUploadProxyDisabled()) { - eTag = await UploadHttpClient.putFilePartV2( - fileUploadURL, - uploadChunk, - progressTracker, - ); - } else { - eTag = await UploadHttpClient.putFilePart( - fileUploadURL, - uploadChunk, - progressTracker, - ); - } - partEtags.push({ PartNumber: index + 1, ETag: eTag }); - } - const { done } = await streamReader.read(); - if (!done) { - throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); - } - logger(`uploading file in chunks done`); - logger(`completing multipart upload`); - await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); - logger(`completing multipart upload done`); - return multipartUploadURLs.objectKey; -} - -function getRandomProgressPerPartUpload(uploadPartCount: number) { - const percentPerPart = - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; - return percentPerPart; -} - -async function combineChunksToFormUploadPart( - streamReader: ReadableStreamDefaultReader, -) { - const combinedChunks = []; - for (let i = 0; i < FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { - const { done, value: chunk } = await streamReader.read(); - if (done) { - break; - } - for (let index = 0; index < chunk.length; index++) { - combinedChunks.push(chunk[index]); - } - } - return Uint8Array.from(combinedChunks); -} - -async function completeMultipartUpload( - partEtags: PartEtag[], - completeURL: string, -) { - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: partEtags } }, - options, - ); - if (!uploadService.getIsCFUploadProxyDisabled()) { - await UploadHttpClient.completeMultipartUploadV2(completeURL, body); - } else { - await UploadHttpClient.completeMultipartUpload(completeURL, body); - } -} diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 4a82c448bd..4444a49660 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -11,8 +11,10 @@ import { wait } from "@ente/shared/utils"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { + FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE, + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UPLOAD_RESULT, } from "constants/upload"; import { addToCollection } from "services/collectionService"; @@ -30,6 +32,7 @@ import { FileTypeInfo, FileWithMetadata, Logger, + MultipartUploadURLs, ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, @@ -48,6 +51,7 @@ import { } from "utils/magicMetadata"; import { readStream } from "utils/native-stream"; import { hasFileHash } from "utils/upload"; +import * as convert from "xml-js"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { @@ -55,7 +59,6 @@ import { getLivePhotoFileType, getLivePhotoSize, } from "./metadataService"; -import { uploadStreamUsingMultipart } from "./multiPartUploadService"; import publicUploadHttpClient from "./publicUploadHttpClient"; import { fallbackThumbnail, @@ -909,3 +912,125 @@ function areFilesWithFileHashSame( return existingFile.hash === newFile.hash; } } + +interface PartEtag { + PartNumber: number; + ETag: string; +} + +function calculatePartCount(chunkCount: number) { + const partCount = Math.ceil( + chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + ); + return partCount; +} + +export async function uploadStreamUsingMultipart( + logger: Logger, + fileLocalID: number, + dataStream: DataStream, +) { + const uploadPartCount = calculatePartCount(dataStream.chunkCount); + logger(`fetching ${uploadPartCount} urls for multipart upload`); + const multipartUploadURLs = + await uploadService.fetchMultipartUploadURLs(uploadPartCount); + logger(`fetched ${uploadPartCount} urls for multipart upload`); + + const fileObjectKey = await uploadStreamInParts( + logger, + multipartUploadURLs, + dataStream.stream, + fileLocalID, + uploadPartCount, + ); + return fileObjectKey; +} + +async function uploadStreamInParts( + logger: Logger, + multipartUploadURLs: MultipartUploadURLs, + dataStream: ReadableStream, + fileLocalID: number, + uploadPartCount: number, +) { + const streamReader = dataStream.getReader(); + const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); + const partEtags: PartEtag[] = []; + logger(`uploading file in chunks`); + for (const [ + index, + fileUploadURL, + ] of multipartUploadURLs.partURLs.entries()) { + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } + const uploadChunk = await combineChunksToFormUploadPart(streamReader); + const progressTracker = UIService.trackUploadProgress( + fileLocalID, + percentPerPart, + index, + ); + let eTag = null; + if (!uploadService.getIsCFUploadProxyDisabled()) { + eTag = await UploadHttpClient.putFilePartV2( + fileUploadURL, + uploadChunk, + progressTracker, + ); + } else { + eTag = await UploadHttpClient.putFilePart( + fileUploadURL, + uploadChunk, + progressTracker, + ); + } + partEtags.push({ PartNumber: index + 1, ETag: eTag }); + } + const { done } = await streamReader.read(); + if (!done) { + throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); + } + logger(`uploading file in chunks done`); + logger(`completing multipart upload`); + await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); + logger(`completing multipart upload done`); + return multipartUploadURLs.objectKey; +} + +function getRandomProgressPerPartUpload(uploadPartCount: number) { + const percentPerPart = + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; + return percentPerPart; +} + +async function combineChunksToFormUploadPart( + streamReader: ReadableStreamDefaultReader, +) { + const combinedChunks = []; + for (let i = 0; i < FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART; i++) { + const { done, value: chunk } = await streamReader.read(); + if (done) { + break; + } + for (let index = 0; index < chunk.length; index++) { + combinedChunks.push(chunk[index]); + } + } + return Uint8Array.from(combinedChunks); +} + +async function completeMultipartUpload( + partEtags: PartEtag[], + completeURL: string, +) { + const options = { compact: true, ignoreComment: true, spaces: 4 }; + const body = convert.js2xml( + { CompleteMultipartUpload: { Part: partEtags } }, + options, + ); + if (!uploadService.getIsCFUploadProxyDisabled()) { + await UploadHttpClient.completeMultipartUploadV2(completeURL, body); + } else { + await UploadHttpClient.completeMultipartUpload(completeURL, body); + } +} diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index c75cc606bc..33446e4e08 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,10 +1,8 @@ import { basename, dirname } from "@/next/file"; import { ElectronFile } from "@/next/types/file"; -import { FILE_TYPE } from "constants/file"; import { PICKED_UPLOAD_TYPE } from "constants/upload"; import isElectron from "is-electron"; import { exportMetadataDirectoryName } from "services/export"; -import { EnteFile } from "types/file"; import { FileWithCollection, Metadata, From 190dc586a9bba60befa488e6930e9a293a761ac2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 18:41:10 +0530 Subject: [PATCH 40/82] Prune --- web/apps/cast/src/constants/collection.ts | 24 ---- .../{collection/index.ts => collection.ts} | 75 ++---------- web/apps/cast/src/types/file/index.ts | 19 ---- web/apps/cast/src/types/upload.ts | 25 ++++ web/apps/cast/src/types/upload/index.ts | 107 ------------------ .../src/services/upload/uploadService.ts | 11 +- 6 files changed, 36 insertions(+), 225 deletions(-) delete mode 100644 web/apps/cast/src/constants/collection.ts rename web/apps/cast/src/types/{collection/index.ts => collection.ts} (61%) create mode 100644 web/apps/cast/src/types/upload.ts delete mode 100644 web/apps/cast/src/types/upload/index.ts diff --git a/web/apps/cast/src/constants/collection.ts b/web/apps/cast/src/constants/collection.ts deleted file mode 100644 index d91cfc81d8..0000000000 --- a/web/apps/cast/src/constants/collection.ts +++ /dev/null @@ -1,24 +0,0 @@ -export enum CollectionType { - folder = "folder", - favorites = "favorites", - album = "album", - uncategorized = "uncategorized", -} - -export enum CollectionSummaryType { - folder = "folder", - favorites = "favorites", - album = "album", - archive = "archive", - trash = "trash", - uncategorized = "uncategorized", - all = "all", - outgoingShare = "outgoingShare", - incomingShareViewer = "incomingShareViewer", - incomingShareCollaborator = "incomingShareCollaborator", - sharedOnlyViaLink = "sharedOnlyViaLink", - archived = "archived", - defaultHidden = "defaultHidden", - hiddenItems = "hiddenItems", - pinned = "pinned", -} diff --git a/web/apps/cast/src/types/collection/index.ts b/web/apps/cast/src/types/collection.ts similarity index 61% rename from web/apps/cast/src/types/collection/index.ts rename to web/apps/cast/src/types/collection.ts index f9ea9ef04b..c495937ae0 100644 --- a/web/apps/cast/src/types/collection/index.ts +++ b/web/apps/cast/src/types/collection.ts @@ -1,4 +1,3 @@ -import { CollectionSummaryType, CollectionType } from "constants/collection"; import { EnteFile } from "types/file"; import { EncryptedMagicMetadata, @@ -20,6 +19,13 @@ export interface CollectionUser { role: COLLECTION_ROLE; } +enum CollectionType { + folder = "folder", + favorites = "favorites", + album = "album", + uncategorized = "uncategorized", +} + export interface EncryptedCollection { id: number; owner: CollectionUser; @@ -32,7 +38,7 @@ export interface EncryptedCollection { type: CollectionType; attributes: collectionAttributes; sharees: CollectionUser[]; - publicURLs?: PublicURL[]; + publicURLs?: unknown; updationTime: number; isDeleted: boolean; magicMetadata: EncryptedMagicMetadata; @@ -61,54 +67,6 @@ export interface Collection // define a method on Collection interface to return the sync key as collection.id-time // this is used to store the last sync time of a collection in local storage -export interface PublicURL { - url: string; - deviceLimit: number; - validTill: number; - enableDownload: boolean; - enableCollect: boolean; - passwordEnabled: boolean; - nonce?: string; - opsLimit?: number; - memLimit?: number; -} - -export interface UpdatePublicURL { - collectionID: number; - disablePassword?: boolean; - enableDownload?: boolean; - enableCollect?: boolean; - validTill?: number; - deviceLimit?: number; - passHash?: string; - nonce?: string; - opsLimit?: number; - memLimit?: number; -} - -export interface CreatePublicAccessTokenRequest { - collectionID: number; - validTill?: number; - deviceLimit?: number; -} - -export interface EncryptedFileKey { - id: number; - encryptedKey: string; - keyDecryptionNonce: string; -} - -export interface AddToCollectionRequest { - collectionID: number; - files: EncryptedFileKey[]; -} - -export interface MoveToCollectionRequest { - fromCollectionID: number; - toCollectionID: number; - files: EncryptedFileKey[]; -} - export interface collectionAttributes { encryptedPath?: string; pathDecryptionNonce?: string; @@ -116,11 +74,6 @@ export interface collectionAttributes { export type CollectionToFileMap = Map; -export interface RemoveFromCollectionRequest { - collectionID: number; - fileIDs: number[]; -} - export interface CollectionMagicMetadataProps { visibility?: VISIBILITY_STATE; subType?: SUB_TYPE; @@ -144,16 +97,4 @@ export interface CollectionPublicMagicMetadataProps { export type CollectionPublicMagicMetadata = MagicMetadataCore; -export interface CollectionSummary { - id: number; - name: string; - type: CollectionSummaryType; - coverFile: EnteFile; - latestFile: EnteFile; - fileCount: number; - updationTime: number; - order?: number; -} - -export type CollectionSummaries = Map; export type CollectionFilesCount = Map; diff --git a/web/apps/cast/src/types/file/index.ts b/web/apps/cast/src/types/file/index.ts index 1813b5416d..f6b868c55b 100644 --- a/web/apps/cast/src/types/file/index.ts +++ b/web/apps/cast/src/types/file/index.ts @@ -64,25 +64,6 @@ export interface EnteFile isConverted?: boolean; } -export interface TrashRequest { - items: TrashRequestItems[]; -} - -export interface TrashRequestItems { - fileID: number; - collectionID: number; -} - -export interface FileWithUpdatedMagicMetadata { - file: EnteFile; - updatedMagicMetadata: FileMagicMetadata; -} - -export interface FileWithUpdatedPublicMagicMetadata { - file: EnteFile; - updatedPublicMagicMetadata: FilePublicMagicMetadata; -} - export interface FileMagicMetadataProps { visibility?: VISIBILITY_STATE; filePaths?: string[]; diff --git a/web/apps/cast/src/types/upload.ts b/web/apps/cast/src/types/upload.ts new file mode 100644 index 0000000000..6dc8818202 --- /dev/null +++ b/web/apps/cast/src/types/upload.ts @@ -0,0 +1,25 @@ +import { FILE_TYPE } from "constants/file"; + +export interface Metadata { + title: string; + creationTime: number; + modificationTime: number; + latitude: number; + longitude: number; + fileType: FILE_TYPE; + hasStaticThumbnail?: boolean; + hash?: string; + imageHash?: string; + videoHash?: string; + localID?: number; + version?: number; + deviceFolder?: string; +} + +export interface FileTypeInfo { + fileType: FILE_TYPE; + exactType: string; + mimeType?: string; + imageType?: string; + videoType?: string; +} diff --git a/web/apps/cast/src/types/upload/index.ts b/web/apps/cast/src/types/upload/index.ts deleted file mode 100644 index 0e249846ad..0000000000 --- a/web/apps/cast/src/types/upload/index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { - B64EncryptionResult, - LocalFileAttributes, -} from "@ente/shared/crypto/types"; -import { FILE_TYPE } from "constants/file"; -import { - FilePublicMagicMetadata, - FilePublicMagicMetadataProps, - MetadataFileAttributes, - S3FileAttributes, -} from "types/file"; -import { EncryptedMagicMetadata } from "types/magicMetadata"; - -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - -export function isDataStream(object: any): object is DataStream { - return "stream" in object; -} - -export type Logger = (message: string) => void; - -export interface Metadata { - title: string; - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; - fileType: FILE_TYPE; - hasStaticThumbnail?: boolean; - hash?: string; - imageHash?: string; - videoHash?: string; - localID?: number; - version?: number; - deviceFolder?: string; -} - -export interface FileTypeInfo { - fileType: FILE_TYPE; - exactType: string; - mimeType?: string; - imageType?: string; - videoType?: string; -} - -export interface UploadURL { - url: string; - objectKey: string; -} - -export interface FileInMemory { - filedata: Uint8Array | DataStream; - thumbnail: Uint8Array; - hasStaticThumbnail: boolean; -} - -export interface FileWithMetadata - extends Omit { - metadata: Metadata; - localID: number; - pubMagicMetadata: FilePublicMagicMetadata; -} - -export interface EncryptedFile { - file: ProcessedFile; - fileKey: B64EncryptionResult; -} -export interface ProcessedFile { - file: LocalFileAttributes; - thumbnail: LocalFileAttributes; - metadata: LocalFileAttributes; - pubMagicMetadata: EncryptedMagicMetadata; - localID: number; -} -export interface BackupedFile { - file: S3FileAttributes; - thumbnail: S3FileAttributes; - metadata: MetadataFileAttributes; - pubMagicMetadata: EncryptedMagicMetadata; -} - -export interface UploadFile extends BackupedFile { - collectionID: number; - encryptedKey: string; - keyDecryptionNonce: string; -} - -export interface ParsedExtractedMetadata { - location: Location; - creationTime: number; - width: number; - height: number; -} - -export interface PublicUploadProps { - token: string; - passwordToken: string; - accessedThroughSharedURL: boolean; -} - -export interface ExtractMetadataResult { - metadata: Metadata; - publicMagicMetadata: FilePublicMagicMetadataProps; -} diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 4444a49660..548812e18d 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -918,19 +918,14 @@ interface PartEtag { ETag: string; } -function calculatePartCount(chunkCount: number) { - const partCount = Math.ceil( - chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, - ); - return partCount; -} - export async function uploadStreamUsingMultipart( logger: Logger, fileLocalID: number, dataStream: DataStream, ) { - const uploadPartCount = calculatePartCount(dataStream.chunkCount); + const uploadPartCount = Math.ceil( + dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, + ); logger(`fetching ${uploadPartCount} urls for multipart upload`); const multipartUploadURLs = await uploadService.fetchMultipartUploadURLs(uploadPartCount); From f96adddf54921ffa171ed48df437d0fb799897fd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 18:54:34 +0530 Subject: [PATCH 41/82] Prune --- web/apps/photos/src/constants/upload.ts | 5 -- .../src/services/upload/metadataService.ts | 6 +-- .../src/services/upload/uploadService.ts | 5 +- web/apps/photos/src/types/upload/index.ts | 20 ++++---- web/packages/next/types/file.ts | 46 ------------------- 5 files changed, 15 insertions(+), 67 deletions(-) diff --git a/web/apps/photos/src/constants/upload.ts b/web/apps/photos/src/constants/upload.ts index e1ee197bcf..1c677470ef 100644 --- a/web/apps/photos/src/constants/upload.ts +++ b/web/apps/photos/src/constants/upload.ts @@ -70,11 +70,6 @@ export enum UPLOAD_STAGES { FINISH, } -export enum UPLOAD_STRATEGY { - SINGLE_COLLECTION, - COLLECTION_PER_FOLDER, -} - export enum UPLOAD_RESULT { FAILED, ALREADY_UPLOADED, diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index 3944e6fa51..f7a3c02646 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,5 +1,5 @@ import { ensureElectron } from "@/next/electron"; -import { basename, getFileNameSize } from "@/next/file"; +import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; @@ -17,7 +17,6 @@ import { getElectronFileStream, getFileStream } from "services/readerService"; import { getFileType } from "services/typeDetectionService"; import { FilePublicMagicMetadataProps } from "types/file"; import { - DataStream, ExtractMetadataResult, FileTypeInfo, LivePhotoAssets, @@ -26,6 +25,7 @@ import { ParsedExtractedMetadata, ParsedMetadataJSON, ParsedMetadataJSONMap, + type DataStream, type FileWithCollection, type FileWithCollection2, type LivePhotoAssets2, @@ -34,7 +34,7 @@ import { import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; import uploadCancelService from "./uploadCancelService"; -import { extractFileMetadata, getFileName } from "./uploadService"; +import { getFileName } from "./uploadService"; const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { creationTime: null, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 548812e18d..996464ab6f 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -26,12 +26,10 @@ import { import { EncryptedMagicMetadata } from "types/magicMetadata"; import { BackupedFile, - DataStream, EncryptedFile, FileInMemory, FileTypeInfo, FileWithMetadata, - Logger, MultipartUploadURLs, ParsedMetadataJSONMap, ProcessedFile, @@ -40,6 +38,7 @@ import { UploadFile, UploadURL, isDataStream, + type DataStream, type FileWithCollection2, type LivePhotoAssets2, type Metadata, @@ -250,6 +249,8 @@ const uploadService = new UploadService(); export default uploadService; +export type Logger = (message: string) => void; + interface UploadResponse { fileUploadResult: UPLOAD_RESULT; uploadedFile?: EnteFile; diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 00bf7c8b86..674aede6a3 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -13,17 +13,6 @@ import { } from "types/file"; import { EncryptedMagicMetadata } from "types/magicMetadata"; -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - -export function isDataStream(object: any): object is DataStream { - return "stream" in object; -} - -export type Logger = (message: string) => void; - export interface Metadata { /** * The file name. @@ -112,6 +101,15 @@ export interface UploadURL { objectKey: string; } +export interface DataStream { + stream: ReadableStream; + chunkCount: number; +} + +export function isDataStream(object: any): object is DataStream { + return "stream" in object; +} + export interface FileInMemory { filedata: Uint8Array | DataStream; /** The JPEG data of the generated thumbnail */ diff --git a/web/packages/next/types/file.ts b/web/packages/next/types/file.ts index 17f122f051..75641e3a27 100644 --- a/web/packages/next/types/file.ts +++ b/web/packages/next/types/file.ts @@ -1,10 +1,3 @@ -import type { Electron } from "./ipc"; - -export enum UPLOAD_STRATEGY { - SINGLE_COLLECTION, - COLLECTION_PER_FOLDER, -} - /* * ElectronFile is a custom interface that is used to represent * any file on disk as a File-like object in the Electron desktop app. @@ -23,45 +16,6 @@ export interface ElectronFile { arrayBuffer: () => Promise; } -/** - * A file path that we obtain from the Node.js layer of our desktop app. - * - * When a user drags and drops or otherwise interactively provides us with a - * file, we get an object that conforms to the [Web File - * API](https://developer.mozilla.org/en-US/docs/Web/API/File). - * - * However, we cannot programmatically create such File objects to arbitrary - * absolute paths on user's local filesystem for security reasons. - * - * This restricts us in cases where the user does want us to, say, watch a - * folder on disk for changes, or auto-resume previously interrupted uploads - * when the app gets restarted. - * - * For such functionality, we defer to our Node.js layer via the - * {@link Electron} object. This IPC communication works with absolute paths of - * disk files or folders, and the native Node.js layer can then perform the - * relevant operations on them. - * - * The {@link DesktopFilePath} interface bundles such a absolute {@link path} - * with an {@link Electron} object that we can later use to, say, read or write - * to that file by using the IPC methods. - * - * This is the same electron instance as `globalThis.electron`, except it is - * non-optional here. Thus we're guaranteed that whatever code is passing us an - * absolute file path is running in the context of our desktop app. - */ -export interface DesktopFilePath { - /** The absolute path to a file or a folder on the local filesystem. */ - path: string; - /** The {@link Electron} instance that we can use to operate on the path. */ - electron: Electron; -} - -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - export interface EventQueueItem { type: "upload" | "trash"; folderPath: string; From e81d3a0c3cd57e84328508834680550f8f73b10a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 18:56:15 +0530 Subject: [PATCH 42/82] Remove tracer --- .../src/services/upload/uploadService.ts | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 996464ab6f..1027d7c5da 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -109,45 +109,34 @@ class UploadService { this.pendingUploadCount--; } - async uploadToBucket( - logger: Logger, - file: ProcessedFile, - ): Promise { + async uploadToBucket(file: ProcessedFile): Promise { try { let fileObjectKey: string = null; - logger("uploading file to bucket"); + if (isDataStream(file.file.encryptedData)) { - logger("uploading using multipart"); fileObjectKey = await uploadStreamUsingMultipart( - logger, file.localID, file.file.encryptedData, ); - logger("uploading using multipart done"); } else { - logger("uploading using single part"); const progressTracker = UIService.trackUploadProgress( file.localID, ); const fileUploadURL = await this.getUploadURL(); if (!this.isCFUploadProxyDisabled) { - logger("uploading using cf proxy"); fileObjectKey = await UploadHttpClient.putFileV2( fileUploadURL, file.file.encryptedData as Uint8Array, progressTracker, ); } else { - logger("uploading directly to s3"); fileObjectKey = await UploadHttpClient.putFile( fileUploadURL, file.file.encryptedData as Uint8Array, progressTracker, ); } - logger("uploading using single part done"); } - logger("uploading thumbnail to bucket"); const thumbnailUploadURL = await this.getUploadURL(); let thumbnailObjectKey: string = null; if (!this.isCFUploadProxyDisabled) { @@ -163,7 +152,6 @@ class UploadService { null, ); } - logger("uploading thumbnail to bucket done"); const backupedFile: BackupedFile = { file: { @@ -249,8 +237,6 @@ const uploadService = new UploadService(); export default uploadService; -export type Logger = (message: string) => void; - interface UploadResponse { fileUploadResult: UPLOAD_RESULT; uploadedFile?: EnteFile; @@ -380,11 +366,7 @@ export async function uploader( throw Error(CustomError.UPLOAD_CANCELLED); } log.info(`uploadToBucket ${fileNameSize}`); - const logger: Logger = (message: string) => { - log.info(message, `fileNameSize: ${fileNameSize}`); - }; const backupedFile: BackupedFile = await uploadService.uploadToBucket( - logger, encryptedFile.file, ); @@ -920,20 +902,16 @@ interface PartEtag { } export async function uploadStreamUsingMultipart( - logger: Logger, fileLocalID: number, dataStream: DataStream, ) { const uploadPartCount = Math.ceil( dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, ); - logger(`fetching ${uploadPartCount} urls for multipart upload`); const multipartUploadURLs = await uploadService.fetchMultipartUploadURLs(uploadPartCount); - logger(`fetched ${uploadPartCount} urls for multipart upload`); const fileObjectKey = await uploadStreamInParts( - logger, multipartUploadURLs, dataStream.stream, fileLocalID, @@ -943,7 +921,6 @@ export async function uploadStreamUsingMultipart( } async function uploadStreamInParts( - logger: Logger, multipartUploadURLs: MultipartUploadURLs, dataStream: ReadableStream, fileLocalID: number, @@ -952,7 +929,6 @@ async function uploadStreamInParts( const streamReader = dataStream.getReader(); const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); const partEtags: PartEtag[] = []; - logger(`uploading file in chunks`); for (const [ index, fileUploadURL, @@ -986,10 +962,7 @@ async function uploadStreamInParts( if (!done) { throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); } - logger(`uploading file in chunks done`); - logger(`completing multipart upload`); await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); - logger(`completing multipart upload done`); return multipartUploadURLs.objectKey; } From 0ca4b06872ce51b7bef1f1400caf779dbff4e485 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 19:46:21 +0530 Subject: [PATCH 43/82] Trim logging --- .../src/services/upload/uploadService.ts | 41 +++++-------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 1027d7c5da..d3c1728d58 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -249,14 +249,12 @@ export async function uploader( parsedMetadataJSONMap: ParsedMetadataJSONMap, uploaderName: string, ): Promise { + const name = assetName(fileWithCollection); + log.info(`Uploading ${name}`); + const { collection, localID, ...uploadAsset2 } = fileWithCollection; /* TODO(MR): ElectronFile changes */ const uploadAsset = uploadAsset2 as UploadAsset; - const fileNameSize = `${assetName( - fileWithCollection, - )}_${convertBytesToHumanReadable(getAssetSize(uploadAsset))}`; - - log.info(`uploader called for ${fileNameSize}`); UIService.setFileProgress(localID, 0); await wait(0); let fileTypeInfo: FileTypeInfo; @@ -268,14 +266,8 @@ export async function uploader( if (fileSize >= maxFileSize) { return { fileUploadResult: UPLOAD_RESULT.TOO_LARGE }; } - log.info(`getting filetype for ${fileNameSize}`); fileTypeInfo = await getAssetFileType(uploadAsset); - log.info( - `got filetype for ${fileNameSize} - ${JSON.stringify(fileTypeInfo)}`, - ); - - log.info(`extracting metadata ${fileNameSize}`); const { metadata, publicMagicMetadata } = await extractAssetMetadata( worker, parsedMetadataJSONMap, @@ -288,23 +280,14 @@ export async function uploader( existingFiles, metadata, ); - log.debug( - () => - `matchedFileList: ${matchingExistingFiles - .map((f) => `${f.id}-${f.metadata.title}`) - .join(",")}`, - ); + if (matchingExistingFiles?.length) { const matchingExistingFilesCollectionIDs = matchingExistingFiles.map((e) => e.collectionID); - log.debug( - () => - `matched file collectionIDs:${matchingExistingFilesCollectionIDs} - and collectionID:${collection.id}`, - ); + if (matchingExistingFilesCollectionIDs.includes(collection.id)) { log.info( - `file already present in the collection , skipped upload for ${fileNameSize}`, + `Skipping upload of ${name} since it is already present in the collection`, ); const sameCollectionMatchingExistingFile = matchingExistingFiles.find( @@ -316,7 +299,7 @@ export async function uploader( }; } else { log.info( - `same file in ${matchingExistingFilesCollectionIDs.length} collection found for ${fileNameSize} ,adding symlink`, + `Symlinking ${name} to existing file in ${matchingExistingFilesCollectionIDs.length} collections`, ); // any of the matching file can used to add a symlink const resultFile = Object.assign({}, matchingExistingFiles[0]); @@ -331,7 +314,6 @@ export async function uploader( if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); } - log.info(`reading asset ${fileNameSize}`); const file = await readAsset(fileTypeInfo, uploadAsset); @@ -355,7 +337,7 @@ export async function uploader( if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); } - log.info(`encryptFile ${fileNameSize}`); + const encryptedFile = await encryptFile( worker, fileWithMetadata, @@ -365,7 +347,7 @@ export async function uploader( if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); } - log.info(`uploadToBucket ${fileNameSize}`); + const backupedFile: BackupedFile = await uploadService.uploadToBucket( encryptedFile.file, ); @@ -376,12 +358,9 @@ export async function uploader( keyDecryptionNonce: encryptedFile.fileKey.nonce, ...backupedFile, }; - log.info(`uploading file to server ${fileNameSize}`); const uploadedFile = await uploadService.uploadFile(uploadFile); - log.info(`${fileNameSize} successfully uploaded`); - return { fileUploadResult: metadata.hasStaticThumbnail ? UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL @@ -389,7 +368,7 @@ export async function uploader( uploadedFile: uploadedFile, }; } catch (e) { - log.info(`upload failed for ${fileNameSize} ,error: ${e.message}`); + log.error(`Upload failed for ${name}`, e); if ( e.message !== CustomError.UPLOAD_CANCELLED && e.message !== CustomError.UNSUPPORTED_FILE_FORMAT From 6b55f3b2f1fef13526ea41f33c84934cd4307968 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 19:50:49 +0530 Subject: [PATCH 44/82] Reduce use of uiservice --- web/apps/photos/src/services/upload/uploadManager.ts | 5 +++++ web/apps/photos/src/services/upload/uploadService.ts | 8 +++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 0b9b485971..5345887e59 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -6,6 +6,7 @@ import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; +import { wait } from "@ente/shared/utils"; import { Remote } from "comlink"; import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload"; import isElectron from "is-electron"; @@ -372,6 +373,10 @@ class UploadManager { const { collectionID } = fileWithCollection; const collection = this.collections.get(collectionID); fileWithCollection = { ...fileWithCollection, collection }; + + UIService.setFileProgress(fileWithCollection.localID, 0); + await wait(0); + const { fileUploadResult, uploadedFile } = await uploader( worker, this.existingFiles, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index d3c1728d58..6aecddcf8a 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -242,21 +242,19 @@ interface UploadResponse { uploadedFile?: EnteFile; } -export async function uploader( +export const uploader = async ( worker: Remote, existingFiles: EnteFile[], fileWithCollection: FileWithCollection2, parsedMetadataJSONMap: ParsedMetadataJSONMap, uploaderName: string, -): Promise { +): Promise => { const name = assetName(fileWithCollection); log.info(`Uploading ${name}`); const { collection, localID, ...uploadAsset2 } = fileWithCollection; /* TODO(MR): ElectronFile changes */ const uploadAsset = uploadAsset2 as UploadAsset; - UIService.setFileProgress(localID, 0); - await wait(0); let fileTypeInfo: FileTypeInfo; let fileSize: number; try { @@ -396,7 +394,7 @@ export async function uploader( return { fileUploadResult: UPLOAD_RESULT.FAILED }; } } -} +}; export const getFileName = (file: File | ElectronFile | string) => typeof file == "string" ? basename(file) : file.name; From c948b29729f7bf73bd294df39d31fb8789fd7679 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 19:56:07 +0530 Subject: [PATCH 45/82] Inline --- .../photos/src/services/upload/uiService.ts | 218 -------------- .../src/services/upload/uploadManager.ts | 278 ++++++++++++++++-- .../src/services/upload/uploadService.ts | 48 +-- 3 files changed, 276 insertions(+), 268 deletions(-) delete mode 100644 web/apps/photos/src/services/upload/uiService.ts diff --git a/web/apps/photos/src/services/upload/uiService.ts b/web/apps/photos/src/services/upload/uiService.ts deleted file mode 100644 index 13dd780019..0000000000 --- a/web/apps/photos/src/services/upload/uiService.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { CustomError } from "@ente/shared/error"; -import { Canceler } from "axios"; -import { - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, - UPLOAD_RESULT, - UPLOAD_STAGES, -} from "constants/upload"; -import { - FinishedUploads, - InProgressUpload, - InProgressUploads, - ProgressUpdater, - SegregatedFinishedUploads, -} from "types/upload/ui"; -import uploadCancelService from "./uploadCancelService"; - -const REQUEST_TIMEOUT_TIME = 30 * 1000; // 30 sec; -class UIService { - private progressUpdater: ProgressUpdater; - - // UPLOAD LEVEL STATES - private uploadStage: UPLOAD_STAGES = UPLOAD_STAGES.START; - private filenames: Map = new Map(); - private hasLivePhoto: boolean = false; - private uploadProgressView: boolean = false; - - // STAGE LEVEL STATES - private perFileProgress: number; - private filesUploadedCount: number; - private totalFilesCount: number; - private inProgressUploads: InProgressUploads = new Map(); - private finishedUploads: FinishedUploads = new Map(); - - init(progressUpdater: ProgressUpdater) { - this.progressUpdater = progressUpdater; - this.progressUpdater.setUploadStage(this.uploadStage); - this.progressUpdater.setUploadFilenames(this.filenames); - this.progressUpdater.setHasLivePhotos(this.hasLivePhoto); - this.progressUpdater.setUploadProgressView(this.uploadProgressView); - this.progressUpdater.setUploadCounter({ - finished: this.filesUploadedCount, - total: this.totalFilesCount, - }); - this.progressUpdater.setInProgressUploads( - convertInProgressUploadsToList(this.inProgressUploads), - ); - this.progressUpdater.setFinishedUploads( - segregatedFinishedUploadsToList(this.finishedUploads), - ); - } - - reset(count = 0) { - this.setTotalFileCount(count); - this.filesUploadedCount = 0; - this.inProgressUploads = new Map(); - this.finishedUploads = new Map(); - this.updateProgressBarUI(); - } - - setTotalFileCount(count: number) { - this.totalFilesCount = count; - if (count > 0) { - this.perFileProgress = 100 / this.totalFilesCount; - } else { - this.perFileProgress = 0; - } - } - - setFileProgress(key: number, progress: number) { - this.inProgressUploads.set(key, progress); - this.updateProgressBarUI(); - } - - setUploadStage(stage: UPLOAD_STAGES) { - this.uploadStage = stage; - this.progressUpdater.setUploadStage(stage); - } - - setFilenames(filenames: Map) { - this.filenames = filenames; - this.progressUpdater.setUploadFilenames(filenames); - } - - setHasLivePhoto(hasLivePhoto: boolean) { - this.hasLivePhoto = hasLivePhoto; - this.progressUpdater.setHasLivePhotos(hasLivePhoto); - } - - setUploadProgressView(uploadProgressView: boolean) { - this.uploadProgressView = uploadProgressView; - this.progressUpdater.setUploadProgressView(uploadProgressView); - } - - increaseFileUploaded() { - this.filesUploadedCount++; - this.updateProgressBarUI(); - } - - moveFileToResultList(key: number, uploadResult: UPLOAD_RESULT) { - this.finishedUploads.set(key, uploadResult); - this.inProgressUploads.delete(key); - this.updateProgressBarUI(); - } - - hasFilesInResultList() { - const finishedUploadsList = segregatedFinishedUploadsToList( - this.finishedUploads, - ); - for (const x of finishedUploadsList.values()) { - if (x.length > 0) { - return true; - } - } - return false; - } - - private updateProgressBarUI() { - const { - setPercentComplete, - setUploadCounter, - setInProgressUploads, - setFinishedUploads, - } = this.progressUpdater; - setUploadCounter({ - finished: this.filesUploadedCount, - total: this.totalFilesCount, - }); - let percentComplete = - this.perFileProgress * - (this.finishedUploads.size || this.filesUploadedCount); - if (this.inProgressUploads) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for (const [_, progress] of this.inProgressUploads) { - // filter negative indicator values during percentComplete calculation - if (progress < 0) { - continue; - } - percentComplete += (this.perFileProgress * progress) / 100; - } - } - - setPercentComplete(percentComplete); - setInProgressUploads( - convertInProgressUploadsToList(this.inProgressUploads), - ); - setFinishedUploads( - segregatedFinishedUploadsToList(this.finishedUploads), - ); - } - - trackUploadProgress( - fileLocalID: number, - percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), - index = 0, - ) { - const cancel: { exec: Canceler } = { exec: () => {} }; - const cancelTimedOutRequest = () => - cancel.exec(CustomError.REQUEST_TIMEOUT); - - const cancelCancelledUploadRequest = () => - cancel.exec(CustomError.UPLOAD_CANCELLED); - - let timeout = null; - const resetTimeout = () => { - if (timeout) { - clearTimeout(timeout); - } - timeout = setTimeout(cancelTimedOutRequest, REQUEST_TIMEOUT_TIME); - }; - return { - cancel, - onUploadProgress: (event) => { - this.inProgressUploads.set( - fileLocalID, - Math.min( - Math.round( - percentPerPart * index + - (percentPerPart * event.loaded) / event.total, - ), - 98, - ), - ); - this.updateProgressBarUI(); - if (event.loaded === event.total) { - clearTimeout(timeout); - } else { - resetTimeout(); - } - if (uploadCancelService.isUploadCancelationRequested()) { - cancelCancelledUploadRequest(); - } - }, - }; - } -} - -export default new UIService(); - -function convertInProgressUploadsToList(inProgressUploads) { - return [...inProgressUploads.entries()].map( - ([localFileID, progress]) => - ({ - localFileID, - progress, - }) as InProgressUpload, - ); -} - -function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) { - const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads; - for (const [localID, result] of finishedUploads) { - if (!segregatedFinishedUploads.has(result)) { - segregatedFinishedUploads.set(result, []); - } - segregatedFinishedUploads.get(result).push(localID); - } - return segregatedFinishedUploads; -} diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 5345887e59..8873a2ff6c 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -7,8 +7,13 @@ import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worke import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { wait } from "@ente/shared/utils"; +import { Canceler } from "axios"; import { Remote } from "comlink"; -import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload"; +import { + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, + UPLOAD_RESULT, + UPLOAD_STAGES, +} from "constants/upload"; import isElectron from "is-electron"; import { getLocalPublicFiles, @@ -26,7 +31,13 @@ import { PublicUploadProps, type FileWithCollection2, } from "types/upload"; -import { ProgressUpdater } from "types/upload/ui"; +import { + FinishedUploads, + InProgressUpload, + InProgressUploads, + ProgressUpdater, + SegregatedFinishedUploads, +} from "types/upload/ui"; import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file"; import { areFileWithCollectionsSame, @@ -39,7 +50,6 @@ import { getMetadataJSONMapKeyForJSON, parseMetadataJSON, } from "./metadataService"; -import { default as UIService, default as uiService } from "./uiService"; import uploadCancelService from "./uploadCancelService"; import UploadService, { assetName, @@ -50,6 +60,206 @@ import UploadService, { const MAX_CONCURRENT_UPLOADS = 4; +class UIService { + private progressUpdater: ProgressUpdater; + + // UPLOAD LEVEL STATES + private uploadStage: UPLOAD_STAGES = UPLOAD_STAGES.START; + private filenames: Map = new Map(); + private hasLivePhoto: boolean = false; + private uploadProgressView: boolean = false; + + // STAGE LEVEL STATES + private perFileProgress: number; + private filesUploadedCount: number; + private totalFilesCount: number; + private inProgressUploads: InProgressUploads = new Map(); + private finishedUploads: FinishedUploads = new Map(); + + init(progressUpdater: ProgressUpdater) { + this.progressUpdater = progressUpdater; + this.progressUpdater.setUploadStage(this.uploadStage); + this.progressUpdater.setUploadFilenames(this.filenames); + this.progressUpdater.setHasLivePhotos(this.hasLivePhoto); + this.progressUpdater.setUploadProgressView(this.uploadProgressView); + this.progressUpdater.setUploadCounter({ + finished: this.filesUploadedCount, + total: this.totalFilesCount, + }); + this.progressUpdater.setInProgressUploads( + convertInProgressUploadsToList(this.inProgressUploads), + ); + this.progressUpdater.setFinishedUploads( + segregatedFinishedUploadsToList(this.finishedUploads), + ); + } + + reset(count = 0) { + this.setTotalFileCount(count); + this.filesUploadedCount = 0; + this.inProgressUploads = new Map(); + this.finishedUploads = new Map(); + this.updateProgressBarUI(); + } + + setTotalFileCount(count: number) { + this.totalFilesCount = count; + if (count > 0) { + this.perFileProgress = 100 / this.totalFilesCount; + } else { + this.perFileProgress = 0; + } + } + + setFileProgress(key: number, progress: number) { + this.inProgressUploads.set(key, progress); + this.updateProgressBarUI(); + } + + setUploadStage(stage: UPLOAD_STAGES) { + this.uploadStage = stage; + this.progressUpdater.setUploadStage(stage); + } + + setFilenames(filenames: Map) { + this.filenames = filenames; + this.progressUpdater.setUploadFilenames(filenames); + } + + setHasLivePhoto(hasLivePhoto: boolean) { + this.hasLivePhoto = hasLivePhoto; + this.progressUpdater.setHasLivePhotos(hasLivePhoto); + } + + setUploadProgressView(uploadProgressView: boolean) { + this.uploadProgressView = uploadProgressView; + this.progressUpdater.setUploadProgressView(uploadProgressView); + } + + increaseFileUploaded() { + this.filesUploadedCount++; + this.updateProgressBarUI(); + } + + moveFileToResultList(key: number, uploadResult: UPLOAD_RESULT) { + this.finishedUploads.set(key, uploadResult); + this.inProgressUploads.delete(key); + this.updateProgressBarUI(); + } + + hasFilesInResultList() { + const finishedUploadsList = segregatedFinishedUploadsToList( + this.finishedUploads, + ); + for (const x of finishedUploadsList.values()) { + if (x.length > 0) { + return true; + } + } + return false; + } + + private updateProgressBarUI() { + const { + setPercentComplete, + setUploadCounter, + setInProgressUploads, + setFinishedUploads, + } = this.progressUpdater; + setUploadCounter({ + finished: this.filesUploadedCount, + total: this.totalFilesCount, + }); + let percentComplete = + this.perFileProgress * + (this.finishedUploads.size || this.filesUploadedCount); + if (this.inProgressUploads) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const [_, progress] of this.inProgressUploads) { + // filter negative indicator values during percentComplete calculation + if (progress < 0) { + continue; + } + percentComplete += (this.perFileProgress * progress) / 100; + } + } + + setPercentComplete(percentComplete); + setInProgressUploads( + convertInProgressUploadsToList(this.inProgressUploads), + ); + setFinishedUploads( + segregatedFinishedUploadsToList(this.finishedUploads), + ); + } + + trackUploadProgress( + fileLocalID: number, + percentPerPart = RANDOM_PERCENTAGE_PROGRESS_FOR_PUT(), + index = 0, + ) { + const cancel: { exec: Canceler } = { exec: () => {} }; + const cancelTimedOutRequest = () => + cancel.exec(CustomError.REQUEST_TIMEOUT); + + const cancelCancelledUploadRequest = () => + cancel.exec(CustomError.UPLOAD_CANCELLED); + + let timeout = null; + const resetTimeout = () => { + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(cancelTimedOutRequest, 30 * 1000 /* 30 sec */); + }; + return { + cancel, + onUploadProgress: (event) => { + this.inProgressUploads.set( + fileLocalID, + Math.min( + Math.round( + percentPerPart * index + + (percentPerPart * event.loaded) / event.total, + ), + 98, + ), + ); + this.updateProgressBarUI(); + if (event.loaded === event.total) { + clearTimeout(timeout); + } else { + resetTimeout(); + } + if (uploadCancelService.isUploadCancelationRequested()) { + cancelCancelledUploadRequest(); + } + }, + }; + } +} + +function convertInProgressUploadsToList(inProgressUploads) { + return [...inProgressUploads.entries()].map( + ([localFileID, progress]) => + ({ + localFileID, + progress, + }) as InProgressUpload, + ); +} + +function segregatedFinishedUploadsToList(finishedUploads: FinishedUploads) { + const segregatedFinishedUploads = new Map() as SegregatedFinishedUploads; + for (const [localID, result] of finishedUploads) { + if (!segregatedFinishedUploads.has(result)) { + segregatedFinishedUploads.set(result, []); + } + segregatedFinishedUploads.get(result).push(localID); + } + return segregatedFinishedUploads; +} + class UploadManager { private cryptoWorkers = new Array< ComlinkWorker @@ -64,6 +274,7 @@ class UploadManager { private uploadInProgress: boolean; private publicUploadProps: PublicUploadProps; private uploaderName: string; + private uiService: UIService; public async init( progressUpdater: ProgressUpdater, @@ -71,7 +282,8 @@ class UploadManager { publicCollectProps: PublicUploadProps, isCFUploadProxyDisabled: boolean, ) { - UIService.init(progressUpdater); + this.uiService = new UIService(); + this.uiService.init(progressUpdater); const remoteIsCFUploadProxyDisabled = await getDisableCFUploadProxyFlag(); if (remoteIsCFUploadProxyDisabled) { @@ -97,13 +309,13 @@ class UploadManager { prepareForNewUpload() { this.resetState(); - UIService.reset(); + this.uiService.reset(); uploadCancelService.reset(); - UIService.setUploadStage(UPLOAD_STAGES.START); + this.uiService.setUploadStage(UPLOAD_STAGES.START); } showUploadProgressDialog() { - UIService.setUploadProgressView(true); + this.uiService.setUploadProgressView(true); } async updateExistingFilesAndCollections(collections: Collection[]) { @@ -134,7 +346,7 @@ class UploadManager { log.info( `received ${filesWithCollectionToUploadIn.length} files to upload`, ); - uiService.setFilenames( + this.uiService.setFilenames( new Map( filesWithCollectionToUploadIn.map((mediaFile) => [ mediaFile.localID, @@ -147,7 +359,7 @@ class UploadManager { log.info(`has ${metadataJSONFiles.length} metadata json files`); log.info(`has ${mediaFiles.length} media files`); if (metadataJSONFiles.length) { - UIService.setUploadStage( + this.uiService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.parseMetadataJSONFiles(metadataJSONFiles); @@ -162,7 +374,7 @@ class UploadManager { mediaFiles.length !== analysedMediaFiles.length }`, ); - uiService.setFilenames( + this.uiService.setFilenames( new Map( analysedMediaFiles.map((mediaFile) => [ mediaFile.localID, @@ -171,7 +383,7 @@ class UploadManager { ), ); - UIService.setHasLivePhoto( + this.uiService.setHasLivePhoto( mediaFiles.length !== analysedMediaFiles.length, ); @@ -188,14 +400,14 @@ class UploadManager { throw e; } } finally { - UIService.setUploadStage(UPLOAD_STAGES.FINISH); + this.uiService.setUploadStage(UPLOAD_STAGES.FINISH); for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { this.cryptoWorkers[i]?.terminate(); } this.uploadInProgress = false; } try { - if (!UIService.hasFilesInResultList()) { + if (!this.uiService.hasFilesInResultList()) { return true; } else { return false; @@ -221,7 +433,7 @@ class UploadManager { log.info( `received ${filesWithCollectionToUploadIn.length} files to upload`, ); - uiService.setFilenames( + this.uiService.setFilenames( new Map( filesWithCollectionToUploadIn.map((mediaFile) => [ mediaFile.localID, @@ -234,7 +446,7 @@ class UploadManager { log.info(`has ${metadataJSONFiles.length} metadata json files`); log.info(`has ${mediaFiles.length} media files`); if (metadataJSONFiles.length) { - UIService.setUploadStage( + this.uiService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); await this.parseMetadataJSONFiles(metadataJSONFiles); @@ -249,7 +461,7 @@ class UploadManager { mediaFiles.length !== analysedMediaFiles.length }`, ); - uiService.setFilenames( + this.uiService.setFilenames( new Map( analysedMediaFiles.map((mediaFile) => [ mediaFile.localID, @@ -258,7 +470,7 @@ class UploadManager { ), ); - UIService.setHasLivePhoto( + this.uiService.setHasLivePhoto( mediaFiles.length !== analysedMediaFiles.length, ); @@ -275,14 +487,14 @@ class UploadManager { throw e; } } finally { - UIService.setUploadStage(UPLOAD_STAGES.FINISH); + this.uiService.setUploadStage(UPLOAD_STAGES.FINISH); for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { this.cryptoWorkers[i]?.terminate(); } this.uploadInProgress = false; } try { - if (!UIService.hasFilesInResultList()) { + if (!this.uiService.hasFilesInResultList()) { return true; } else { return false; @@ -297,7 +509,7 @@ class UploadManager { try { log.info(`parseMetadataJSONFiles function executed `); - UIService.reset(metadataFiles.length); + this.uiService.reset(metadataFiles.length); for (const { file, collectionID } of metadataFiles) { const name = getFileName(file); @@ -314,7 +526,7 @@ class UploadManager { getMetadataJSONMapKeyForJSON(collectionID, name), parsedMetadataJSON && { ...parsedMetadataJSON }, ); - UIService.increaseFileUploaded(); + this.uiService.increaseFileUploaded(); } log.info(`successfully parsed metadata json file ${name}`); } catch (e) { @@ -345,11 +557,11 @@ class UploadManager { this.remainingFiles = [...this.remainingFiles, ...mediaFiles]; } - UIService.reset(mediaFiles.length); + this.uiService.reset(mediaFiles.length); await UploadService.setFileCount(mediaFiles.length); - UIService.setUploadStage(UPLOAD_STAGES.UPLOADING); + this.uiService.setUploadStage(UPLOAD_STAGES.UPLOADING); const uploadProcesses = []; for ( @@ -365,6 +577,8 @@ class UploadManager { } private async uploadNextFileInQueue(worker: Remote) { + const uiService = this.uiService; + while (this.filesToBeUploaded.length > 0) { if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); @@ -374,7 +588,7 @@ class UploadManager { const collection = this.collections.get(collectionID); fileWithCollection = { ...fileWithCollection, collection }; - UIService.setFileProgress(fileWithCollection.localID, 0); + uiService.setFileProgress(fileWithCollection.localID, 0); await wait(0); const { fileUploadResult, uploadedFile } = await uploader( @@ -383,6 +597,16 @@ class UploadManager { fileWithCollection, this.parsedMetadataJSONMap, this.uploaderName, + ( + fileLocalID: number, + percentPerPart?: number, + index?: number, + ) => + uiService.trackUploadProgress( + fileLocalID, + percentPerPart, + index, + ), ); const finalUploadResult = await this.postUploadTask( @@ -391,11 +615,11 @@ class UploadManager { fileWithCollection, ); - UIService.moveFileToResultList( + this.uiService.moveFileToResultList( fileWithCollection.localID, finalUploadResult, ); - UIService.increaseFileUploaded(); + this.uiService.increaseFileUploaded(); UploadService.reducePendingUploadCount(); } } @@ -486,7 +710,7 @@ class UploadManager { public cancelRunningUpload() { log.info("user cancelled running upload"); - UIService.setUploadStage(UPLOAD_STAGES.CANCELLING); + this.uiService.setUploadStage(UPLOAD_STAGES.CANCELLING); uploadCancelService.requestUploadCancelation(); } diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 6aecddcf8a..b7fe333213 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -7,7 +7,6 @@ import { CustomErrorMessage } from "@/next/types/ipc"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; -import { wait } from "@ente/shared/utils"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { @@ -30,7 +29,6 @@ import { FileInMemory, FileTypeInfo, FileWithMetadata, - MultipartUploadURLs, ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, @@ -64,7 +62,6 @@ import { generateThumbnailNative, generateThumbnailWeb, } from "./thumbnail"; -import UIService from "./uiService"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; @@ -109,7 +106,10 @@ class UploadService { this.pendingUploadCount--; } - async uploadToBucket(file: ProcessedFile): Promise { + async uploadToBucket( + file: ProcessedFile, + makeProgessTracker: MakeProgressTracker, + ): Promise { try { let fileObjectKey: string = null; @@ -117,11 +117,10 @@ class UploadService { fileObjectKey = await uploadStreamUsingMultipart( file.localID, file.file.encryptedData, + makeProgessTracker, ); } else { - const progressTracker = UIService.trackUploadProgress( - file.localID, - ); + const progressTracker = makeProgessTracker(file.localID); const fileUploadURL = await this.getUploadURL(); if (!this.isCFUploadProxyDisabled) { fileObjectKey = await UploadHttpClient.putFileV2( @@ -237,6 +236,19 @@ const uploadService = new UploadService(); export default uploadService; +/** + * A function that can be called to obtain a "progressTracker" that then is + * directly fed to axios to both cancel the upload if needed, and update the + * progress status. + * + * Needs more type. + */ +type MakeProgressTracker = ( + fileLocalID: number, + percentPerPart?: number, + index?: number, +) => unknown; + interface UploadResponse { fileUploadResult: UPLOAD_RESULT; uploadedFile?: EnteFile; @@ -248,6 +260,7 @@ export const uploader = async ( fileWithCollection: FileWithCollection2, parsedMetadataJSONMap: ParsedMetadataJSONMap, uploaderName: string, + makeProgessTracker: MakeProgressTracker, ): Promise => { const name = assetName(fileWithCollection); log.info(`Uploading ${name}`); @@ -348,6 +361,7 @@ export const uploader = async ( const backupedFile: BackupedFile = await uploadService.uploadToBucket( encryptedFile.file, + makeProgessTracker, ); const uploadFile: UploadFile = { @@ -881,6 +895,7 @@ interface PartEtag { export async function uploadStreamUsingMultipart( fileLocalID: number, dataStream: DataStream, + makeProgessTracker: MakeProgressTracker, ) { const uploadPartCount = Math.ceil( dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, @@ -888,22 +903,9 @@ export async function uploadStreamUsingMultipart( const multipartUploadURLs = await uploadService.fetchMultipartUploadURLs(uploadPartCount); - const fileObjectKey = await uploadStreamInParts( - multipartUploadURLs, - dataStream.stream, - fileLocalID, - uploadPartCount, - ); - return fileObjectKey; -} + const { stream } = dataStream; -async function uploadStreamInParts( - multipartUploadURLs: MultipartUploadURLs, - dataStream: ReadableStream, - fileLocalID: number, - uploadPartCount: number, -) { - const streamReader = dataStream.getReader(); + const streamReader = stream.getReader(); const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); const partEtags: PartEtag[] = []; for (const [ @@ -914,7 +916,7 @@ async function uploadStreamInParts( throw Error(CustomError.UPLOAD_CANCELLED); } const uploadChunk = await combineChunksToFormUploadPart(streamReader); - const progressTracker = UIService.trackUploadProgress( + const progressTracker = makeProgessTracker( fileLocalID, percentPerPart, index, From 7940ef53b55662b42df8d33052081b670cdbbe95 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:15:35 +0530 Subject: [PATCH 46/82] Route --- .../src/services/upload/uploadService.ts | 142 +++++++++--------- 1 file changed, 73 insertions(+), 69 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b7fe333213..6659cd4cbc 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -106,73 +106,6 @@ class UploadService { this.pendingUploadCount--; } - async uploadToBucket( - file: ProcessedFile, - makeProgessTracker: MakeProgressTracker, - ): Promise { - try { - let fileObjectKey: string = null; - - if (isDataStream(file.file.encryptedData)) { - fileObjectKey = await uploadStreamUsingMultipart( - file.localID, - file.file.encryptedData, - makeProgessTracker, - ); - } else { - const progressTracker = makeProgessTracker(file.localID); - const fileUploadURL = await this.getUploadURL(); - if (!this.isCFUploadProxyDisabled) { - fileObjectKey = await UploadHttpClient.putFileV2( - fileUploadURL, - file.file.encryptedData as Uint8Array, - progressTracker, - ); - } else { - fileObjectKey = await UploadHttpClient.putFile( - fileUploadURL, - file.file.encryptedData as Uint8Array, - progressTracker, - ); - } - } - const thumbnailUploadURL = await this.getUploadURL(); - let thumbnailObjectKey: string = null; - if (!this.isCFUploadProxyDisabled) { - thumbnailObjectKey = await UploadHttpClient.putFileV2( - thumbnailUploadURL, - file.thumbnail.encryptedData, - null, - ); - } else { - thumbnailObjectKey = await UploadHttpClient.putFile( - thumbnailUploadURL, - file.thumbnail.encryptedData, - null, - ); - } - - const backupedFile: BackupedFile = { - file: { - decryptionHeader: file.file.decryptionHeader, - objectKey: fileObjectKey, - }, - thumbnail: { - decryptionHeader: file.thumbnail.decryptionHeader, - objectKey: thumbnailObjectKey, - }, - metadata: file.metadata, - pubMagicMetadata: file.pubMagicMetadata, - }; - return backupedFile; - } catch (e) { - if (e.message !== CustomError.UPLOAD_CANCELLED) { - log.error("error uploading to bucket", e); - } - throw e; - } - } - private async getUploadURL() { if (this.uploadURLs.length === 0 && this.pendingUploadCount) { await this.fetchUploadURLs(); @@ -359,9 +292,10 @@ export const uploader = async ( throw Error(CustomError.UPLOAD_CANCELLED); } - const backupedFile: BackupedFile = await uploadService.uploadToBucket( + const backupedFile: BackupedFile = await uploadToBucket( encryptedFile.file, makeProgessTracker, + uploadService.getIsCFUploadProxyDisabled(), ); const uploadFile: UploadFile = { @@ -887,6 +821,75 @@ function areFilesWithFileHashSame( } } +const uploadToBucket = async ( + file: ProcessedFile, + makeProgessTracker: MakeProgressTracker, + isCFUploadProxyDisabled: boolean, +): Promise => { + try { + let fileObjectKey: string = null; + + if (isDataStream(file.file.encryptedData)) { + fileObjectKey = await uploadStreamUsingMultipart( + file.localID, + file.file.encryptedData, + makeProgessTracker, + isCFUploadProxyDisabled, + ); + } else { + const progressTracker = makeProgessTracker(file.localID); + const fileUploadURL = await this.getUploadURL(); + if (!isCFUploadProxyDisabled) { + fileObjectKey = await UploadHttpClient.putFileV2( + fileUploadURL, + file.file.encryptedData as Uint8Array, + progressTracker, + ); + } else { + fileObjectKey = await UploadHttpClient.putFile( + fileUploadURL, + file.file.encryptedData as Uint8Array, + progressTracker, + ); + } + } + const thumbnailUploadURL = await this.getUploadURL(); + let thumbnailObjectKey: string = null; + if (!isCFUploadProxyDisabled) { + thumbnailObjectKey = await UploadHttpClient.putFileV2( + thumbnailUploadURL, + file.thumbnail.encryptedData, + null, + ); + } else { + thumbnailObjectKey = await UploadHttpClient.putFile( + thumbnailUploadURL, + file.thumbnail.encryptedData, + null, + ); + } + + const backupedFile: BackupedFile = { + file: { + decryptionHeader: file.file.decryptionHeader, + objectKey: fileObjectKey, + }, + thumbnail: { + decryptionHeader: file.thumbnail.decryptionHeader, + objectKey: thumbnailObjectKey, + }, + metadata: file.metadata, + pubMagicMetadata: file.pubMagicMetadata, + }; + return backupedFile; + } catch (e) { + if (e.message !== CustomError.UPLOAD_CANCELLED) { + log.error("error uploading to bucket", e); + } + throw e; + } +}; + interface PartEtag { PartNumber: number; ETag: string; @@ -896,6 +899,7 @@ export async function uploadStreamUsingMultipart( fileLocalID: number, dataStream: DataStream, makeProgessTracker: MakeProgressTracker, + isCFUploadProxyDisabled: boolean, ) { const uploadPartCount = Math.ceil( dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, @@ -922,7 +926,7 @@ export async function uploadStreamUsingMultipart( index, ); let eTag = null; - if (!uploadService.getIsCFUploadProxyDisabled()) { + if (!isCFUploadProxyDisabled) { eTag = await UploadHttpClient.putFilePartV2( fileUploadURL, uploadChunk, From 2eef50a8492625742451c210ac6abc45621e6c71 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:20:14 +0530 Subject: [PATCH 47/82] More --- .../src/services/upload/uploadService.ts | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 6659cd4cbc..51217b0d81 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -838,7 +838,7 @@ const uploadToBucket = async ( ); } else { const progressTracker = makeProgessTracker(file.localID); - const fileUploadURL = await this.getUploadURL(); + const fileUploadURL = await uploadService.getUploadURL(); if (!isCFUploadProxyDisabled) { fileObjectKey = await UploadHttpClient.putFileV2( fileUploadURL, @@ -895,7 +895,7 @@ interface PartEtag { ETag: string; } -export async function uploadStreamUsingMultipart( +async function uploadStreamUsingMultipart( fileLocalID: number, dataStream: DataStream, makeProgessTracker: MakeProgressTracker, @@ -910,7 +910,8 @@ export async function uploadStreamUsingMultipart( const { stream } = dataStream; const streamReader = stream.getReader(); - const percentPerPart = getRandomProgressPerPartUpload(uploadPartCount); + const percentPerPart = + RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; const partEtags: PartEtag[] = []; for (const [ index, @@ -945,14 +946,19 @@ export async function uploadStreamUsingMultipart( if (!done) { throw Error(CustomError.CHUNK_MORE_THAN_EXPECTED); } - await completeMultipartUpload(partEtags, multipartUploadURLs.completeURL); - return multipartUploadURLs.objectKey; -} -function getRandomProgressPerPartUpload(uploadPartCount: number) { - const percentPerPart = - RANDOM_PERCENTAGE_PROGRESS_FOR_PUT() / uploadPartCount; - return percentPerPart; + const completeURL = multipartUploadURLs.completeURL; + const cBody = convert.js2xml( + { CompleteMultipartUpload: { Part: partEtags } }, + { compact: true, ignoreComment: true, spaces: 4 }, + ); + if (!isCFUploadProxyDisabled) { + await UploadHttpClient.completeMultipartUploadV2(completeURL, cBody); + } else { + await UploadHttpClient.completeMultipartUpload(completeURL, cBody); + } + + return multipartUploadURLs.objectKey; } async function combineChunksToFormUploadPart( @@ -970,19 +976,3 @@ async function combineChunksToFormUploadPart( } return Uint8Array.from(combinedChunks); } - -async function completeMultipartUpload( - partEtags: PartEtag[], - completeURL: string, -) { - const options = { compact: true, ignoreComment: true, spaces: 4 }; - const body = convert.js2xml( - { CompleteMultipartUpload: { Part: partEtags } }, - options, - ); - if (!uploadService.getIsCFUploadProxyDisabled()) { - await UploadHttpClient.completeMultipartUploadV2(completeURL, body); - } else { - await UploadHttpClient.completeMultipartUpload(completeURL, body); - } -} From 3c9ef294b0908df8970e6f1e0aa09908afb58db8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:21:04 +0530 Subject: [PATCH 48/82] Trim --- web/apps/photos/src/services/upload/uploadService.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 51217b0d81..3b040a0cb2 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -69,8 +69,6 @@ import UploadHttpClient from "./uploadHttpClient"; class UploadService { private uploadURLs: UploadURL[] = []; - private uploaderName: string; - private pendingUploadCount: number = 0; private publicUploadProps: PublicUploadProps = undefined; @@ -90,14 +88,6 @@ class UploadService { await this.preFetchUploadURLs(); } - setUploaderName(uploaderName: string) { - this.uploaderName = uploaderName; - } - - getUploaderName() { - return this.uploaderName; - } - getIsCFUploadProxyDisabled() { return this.isCFUploadProxyDisabled; } From c401b9a9389abffb5ea919edb08d0d32343575d3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:33:06 +0530 Subject: [PATCH 49/82] Log error --- .../src/services/upload/uploadService.ts | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 3b040a0cb2..5fcffa5e80 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,6 +1,6 @@ import { encodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; -import { basename, convertBytesToHumanReadable } from "@/next/file"; +import { basename } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { CustomErrorMessage } from "@/next/types/ipc"; @@ -68,11 +68,8 @@ import UploadHttpClient from "./uploadHttpClient"; /** Upload files to cloud storage */ class UploadService { private uploadURLs: UploadURL[] = []; - private pendingUploadCount: number = 0; - private publicUploadProps: PublicUploadProps = undefined; - private isCFUploadProxyDisabled: boolean = false; init( @@ -304,19 +301,14 @@ export const uploader = async ( uploadedFile: uploadedFile, }; } catch (e) { - log.error(`Upload failed for ${name}`, e); - if ( - e.message !== CustomError.UPLOAD_CANCELLED && - e.message !== CustomError.UNSUPPORTED_FILE_FORMAT - ) { - log.error( - `file upload failed - ${JSON.stringify({ - fileFormat: fileTypeInfo?.exactType, - fileSize: convertBytesToHumanReadable(fileSize), - })}`, - e, - ); + if (e.message == CustomError.UPLOAD_CANCELLED) { + log.info(`Upload for ${name} cancelled`); + } else if (e.message == CustomError.UNSUPPORTED_FILE_FORMAT) { + log.info(`Not uploading ${name}: unsupported file format`); + } else { + log.error(`Upload failed for ${name}`, e); } + const error = handleUploadError(e); switch (error.message) { case CustomError.ETAG_MISSING: From 86a102c47dc15a66ae6af7f94440c9fa1f51be7b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:39:16 +0530 Subject: [PATCH 50/82] Streamlined --- .../src/services/upload/uploadService.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 5fcffa5e80..72d4b6e1f0 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -185,6 +185,11 @@ export const uploader = async ( const name = assetName(fileWithCollection); log.info(`Uploading ${name}`); + const abortIfCancelled = () => { + if (uploadCancelService.isUploadCancelationRequested()) + throw Error(CustomError.UPLOAD_CANCELLED); + }; + const { collection, localID, ...uploadAsset2 } = fileWithCollection; /* TODO(MR): ElectronFile changes */ const uploadAsset = uploadAsset2 as UploadAsset; @@ -242,9 +247,8 @@ export const uploader = async ( }; } } - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } + + abortIfCancelled(); const file = await readAsset(fileTypeInfo, uploadAsset); @@ -265,9 +269,7 @@ export const uploader = async ( pubMagicMetadata, }; - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } + abortIfCancelled(); const encryptedFile = await encryptFile( worker, @@ -275,9 +277,7 @@ export const uploader = async ( collection.key, ); - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } + abortIfCancelled(); const backupedFile: BackupedFile = await uploadToBucket( encryptedFile.file, From e1a3475faa3933f1d5e4a18bd2f0c1b3489f467a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:50:15 +0530 Subject: [PATCH 51/82] Shorten chunk --- .../src/services/upload/uploadService.ts | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 72d4b6e1f0..3a70968308 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -212,38 +212,26 @@ export const uploader = async ( fileTypeInfo, ); - const matchingExistingFiles = findMatchingExistingFiles( - existingFiles, - metadata, - ); + const matches = findMatchingExistingFiles(existingFiles, metadata); + const anyMatch = matches?.length > 0 ? matches[0] : undefined; - if (matchingExistingFiles?.length) { - const matchingExistingFilesCollectionIDs = - matchingExistingFiles.map((e) => e.collectionID); - - if (matchingExistingFilesCollectionIDs.includes(collection.id)) { - log.info( - `Skipping upload of ${name} since it is already present in the collection`, - ); - const sameCollectionMatchingExistingFile = - matchingExistingFiles.find( - (f) => f.collectionID === collection.id, - ); + if (anyMatch) { + const matchInSameCollection = matches.find( + (f) => f.collectionID == collection.id, + ); + if (matchInSameCollection) { return { fileUploadResult: UPLOAD_RESULT.ALREADY_UPLOADED, - uploadedFile: sameCollectionMatchingExistingFile, + uploadedFile: matchInSameCollection, }; } else { - log.info( - `Symlinking ${name} to existing file in ${matchingExistingFilesCollectionIDs.length} collections`, - ); - // any of the matching file can used to add a symlink - const resultFile = Object.assign({}, matchingExistingFiles[0]); - resultFile.collectionID = collection.id; - await addToCollection(collection, [resultFile]); + // Any of the matching files can be used to add a symlink. + const symlink = Object.assign({}, anyMatch); + symlink.collectionID = collection.id; + await addToCollection(collection, [symlink]); return { fileUploadResult: UPLOAD_RESULT.ADDED_SYMLINK, - uploadedFile: resultFile, + uploadedFile: symlink, }; } } From 5befc53d8c177c9f7fc013f440b7a4b23df4d834 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 20:54:57 +0530 Subject: [PATCH 52/82] Streamline --- .../photos/src/services/upload/uploadService.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 3a70968308..ad16fbfe8c 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -240,15 +240,15 @@ export const uploader = async ( const file = await readAsset(fileTypeInfo, uploadAsset); - if (file.hasStaticThumbnail) { - metadata.hasStaticThumbnail = true; - } + if (file.hasStaticThumbnail) metadata.hasStaticThumbnail = true; const pubMagicMetadata = await constructPublicMagicMetadata({ ...publicMagicMetadata, uploaderName, }); + abortIfCancelled(); + const fileWithMetadata: FileWithMetadata = { localID, filedata: file.filedata, @@ -257,8 +257,6 @@ export const uploader = async ( pubMagicMetadata, }; - abortIfCancelled(); - const encryptedFile = await encryptFile( worker, fileWithMetadata, @@ -267,20 +265,18 @@ export const uploader = async ( abortIfCancelled(); - const backupedFile: BackupedFile = await uploadToBucket( + const backupedFile = await uploadToBucket( encryptedFile.file, makeProgessTracker, uploadService.getIsCFUploadProxyDisabled(), ); - const uploadFile: UploadFile = { + const uploadedFile = await uploadService.uploadFile({ collectionID: collection.id, encryptedKey: encryptedFile.fileKey.encryptedData, keyDecryptionNonce: encryptedFile.fileKey.nonce, ...backupedFile, - }; - - const uploadedFile = await uploadService.uploadFile(uploadFile); + }); return { fileUploadResult: metadata.hasStaticThumbnail From 0da46f32982f9c984a0d3dfefa9a08efe9073218 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 21:03:54 +0530 Subject: [PATCH 53/82] Shuffle --- web/apps/photos/src/services/upload/metadataService.ts | 6 +++++- web/apps/photos/src/services/upload/uploadService.ts | 6 +++--- web/apps/photos/src/types/upload/index.ts | 6 +----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index f7a3c02646..f590e50a36 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -17,7 +17,6 @@ import { getElectronFileStream, getFileStream } from "services/readerService"; import { getFileType } from "services/typeDetectionService"; import { FilePublicMagicMetadataProps } from "types/file"; import { - ExtractMetadataResult, FileTypeInfo, LivePhotoAssets, Location, @@ -69,6 +68,11 @@ export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { height: null, }; +export interface ExtractMetadataResult { + metadata: Metadata; + publicMagicMetadata: FilePublicMagicMetadataProps; +} + export async function extractMetadata( worker: Remote, receivedFile: File | ElectronFile, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index ad16fbfe8c..5fb589ca32 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -93,14 +93,14 @@ class UploadService { this.pendingUploadCount--; } - private async getUploadURL() { + async getUploadURL() { if (this.uploadURLs.length === 0 && this.pendingUploadCount) { await this.fetchUploadURLs(); } return this.uploadURLs.pop(); } - public async preFetchUploadURLs() { + private async preFetchUploadURLs() { try { await this.fetchUploadURLs(); // checking for any subscription related errors @@ -819,7 +819,7 @@ const uploadToBucket = async ( ); } } - const thumbnailUploadURL = await this.getUploadURL(); + const thumbnailUploadURL = await uploadService.getUploadURL(); let thumbnailObjectKey: string = null; if (!isCFUploadProxyDisabled) { thumbnailObjectKey = await UploadHttpClient.putFileV2( diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 674aede6a3..81f9a8dc81 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -132,6 +132,7 @@ export interface EncryptedFile { file: ProcessedFile; fileKey: B64EncryptionResult; } + export interface ProcessedFile { file: LocalFileAttributes; thumbnail: LocalFileAttributes; @@ -164,8 +165,3 @@ export interface PublicUploadProps { passwordToken: string; accessedThroughSharedURL: boolean; } - -export interface ExtractMetadataResult { - metadata: Metadata; - publicMagicMetadata: FilePublicMagicMetadataProps; -} From 0d0397124ff5f134225466a822a1d8495af5436a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 09:22:17 +0530 Subject: [PATCH 54/82] Hoist state --- .../src/components/Sidebar/AdvancedSettings.tsx | 10 +++------- .../photos/src/services/upload/uploadManager.ts | 9 +++++++-- .../photos/src/services/upload/uploadService.ts | 14 +++----------- web/apps/photos/src/types/upload/index.ts | 1 - 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx index 6972cc1613..6dc9b851e9 100644 --- a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx +++ b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx @@ -1,4 +1,3 @@ -import log from "@/next/log"; import ChevronRight from "@mui/icons-material/ChevronRight"; import ScienceIcon from "@mui/icons-material/Science"; import { Box, DialogProps, Stack, Typography } from "@mui/material"; @@ -37,13 +36,10 @@ export default function AdvancedSettings({ open, onClose, onRootClose }) { } }; - const toggleCFProxy = async () => { - try { - appContext.setIsCFProxyDisabled(!appContext.isCFProxyDisabled); - } catch (e) { - log.error("toggleFasterUpload failed", e); - } + const toggleCFProxy = () => { + appContext.setIsCFProxyDisabled(!appContext.isCFProxyDisabled); }; + const [indexingStatus, setIndexingStatus] = useState({ indexed: 0, pending: 0, diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 8873a2ff6c..f2a3861924 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -275,21 +275,25 @@ class UploadManager { private publicUploadProps: PublicUploadProps; private uploaderName: string; private uiService: UIService; + private isCFUploadProxyDisabled: boolean = false; + constructor() { + this.uiService = new UIService(); + } public async init( progressUpdater: ProgressUpdater, setFiles: SetFiles, publicCollectProps: PublicUploadProps, isCFUploadProxyDisabled: boolean, ) { - this.uiService = new UIService(); this.uiService.init(progressUpdater); const remoteIsCFUploadProxyDisabled = await getDisableCFUploadProxyFlag(); if (remoteIsCFUploadProxyDisabled) { isCFUploadProxyDisabled = remoteIsCFUploadProxyDisabled; } - UploadService.init(publicCollectProps, isCFUploadProxyDisabled); + this.isCFUploadProxyDisabled = isCFUploadProxyDisabled; + UploadService.init(publicCollectProps); this.setFiles = setFiles; this.publicUploadProps = publicCollectProps; } @@ -597,6 +601,7 @@ class UploadManager { fileWithCollection, this.parsedMetadataJSONMap, this.uploaderName, + this.isCFUploadProxyDisabled, ( fileLocalID: number, percentPerPart?: number, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 5fb589ca32..bd0a3661dd 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -70,14 +70,9 @@ class UploadService { private uploadURLs: UploadURL[] = []; private pendingUploadCount: number = 0; private publicUploadProps: PublicUploadProps = undefined; - private isCFUploadProxyDisabled: boolean = false; - init( - publicUploadProps: PublicUploadProps, - isCFUploadProxyDisabled: boolean, - ) { + init(publicUploadProps: PublicUploadProps) { this.publicUploadProps = publicUploadProps; - this.isCFUploadProxyDisabled = isCFUploadProxyDisabled; } async setFileCount(fileCount: number) { @@ -85,10 +80,6 @@ class UploadService { await this.preFetchUploadURLs(); } - getIsCFUploadProxyDisabled() { - return this.isCFUploadProxyDisabled; - } - reducePendingUploadCount() { this.pendingUploadCount--; } @@ -180,6 +171,7 @@ export const uploader = async ( fileWithCollection: FileWithCollection2, parsedMetadataJSONMap: ParsedMetadataJSONMap, uploaderName: string, + isCFUploadProxyDisabled: boolean, makeProgessTracker: MakeProgressTracker, ): Promise => { const name = assetName(fileWithCollection); @@ -268,7 +260,7 @@ export const uploader = async ( const backupedFile = await uploadToBucket( encryptedFile.file, makeProgessTracker, - uploadService.getIsCFUploadProxyDisabled(), + isCFUploadProxyDisabled, ); const uploadedFile = await uploadService.uploadFile({ diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 81f9a8dc81..98e129bf4f 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -7,7 +7,6 @@ import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { FilePublicMagicMetadata, - FilePublicMagicMetadataProps, MetadataFileAttributes, S3FileAttributes, } from "types/file"; From 56713325ed4c903ce60bc21c4349b16adb0020fb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 09:42:18 +0530 Subject: [PATCH 55/82] Spruce --- .../src/services/upload/uploadService.ts | 99 ++++++++----------- web/apps/photos/src/utils/upload/index.ts | 5 +- 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index bd0a3661dd..442f6e6cdf 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -152,7 +152,7 @@ export default uploadService; * directly fed to axios to both cancel the upload if needed, and update the * progress status. * - * Needs more type. + * Enhancement: The return value needs to be typed. */ type MakeProgressTracker = ( fileLocalID: number, @@ -204,7 +204,10 @@ export const uploader = async ( fileTypeInfo, ); - const matches = findMatchingExistingFiles(existingFiles, metadata); + const matches = existingFiles.filter((file) => + areFilesSame(file.metadata, metadata), + ); + const anyMatch = matches?.length > 0 ? matches[0] : undefined; if (anyMatch) { @@ -717,67 +720,47 @@ async function encryptFileStream( }; } -export function findMatchingExistingFiles( - existingFiles: EnteFile[], - newFileMetadata: Metadata, -): EnteFile[] { - const matchingFiles: EnteFile[] = []; - for (const existingFile of existingFiles) { - if (areFilesSame(existingFile.metadata, newFileMetadata)) { - matchingFiles.push(existingFile); - } - } - return matchingFiles; -} +/** + * Return true if the two files, as represented by their metadata, are same. + * + * Note that the metadata includes the hash of the file's contents (when + * available), so this also in effect compares the contents of the files, not + * just the "meta" information about them. + */ +const areFilesSame = (f: Metadata, g: Metadata) => + hasFileHash(f) && hasFileHash(g) + ? areFilesSameHash(f, g) + : areFilesSameNoHash(f, g); -export function areFilesSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if (hasFileHash(existingFile) && hasFileHash(newFile)) { - return areFilesWithFileHashSame(existingFile, newFile); - } else { - /* - * The maximum difference in the creation/modification times of two similar files is set to 1 second. - * This is because while uploading files in the web - browsers and users could have set reduced - * precision of file times to prevent timing attacks and fingerprinting. - * Context: https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision - */ - const oneSecond = 1e6; - if ( - existingFile.fileType === newFile.fileType && - Math.abs(existingFile.creationTime - newFile.creationTime) < - oneSecond && - Math.abs(existingFile.modificationTime - newFile.modificationTime) < - oneSecond && - existingFile.title === newFile.title - ) { - return true; - } else { - return false; - } - } -} - -function areFilesWithFileHashSame( - existingFile: Metadata, - newFile: Metadata, -): boolean { - if ( - existingFile.fileType !== newFile.fileType || - existingFile.title !== newFile.title - ) { +const areFilesSameHash = (f: Metadata, g: Metadata) => { + if (f.fileType !== g.fileType || f.title !== g.title) { return false; } - if (existingFile.fileType === FILE_TYPE.LIVE_PHOTO) { - return ( - existingFile.imageHash === newFile.imageHash && - existingFile.videoHash === newFile.videoHash - ); + if (f.fileType === FILE_TYPE.LIVE_PHOTO) { + return f.imageHash === g.imageHash && f.videoHash === g.videoHash; } else { - return existingFile.hash === newFile.hash; + return f.hash === g.hash; } -} +}; + +const areFilesSameNoHash = (f: Metadata, g: Metadata) => { + /* + * The maximum difference in the creation/modification times of two similar + * files is set to 1 second. This is because while uploading files in the + * web - browsers and users could have set reduced precision of file times + * to prevent timing attacks and fingerprinting. + * + * See: + * https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision + */ + const oneSecond = 1e6; + return ( + f.fileType === g.fileType && + Math.abs(f.creationTime - g.creationTime) < oneSecond && + Math.abs(f.modificationTime - g.modificationTime) < oneSecond && + f.title === g.title + ); +}; const uploadToBucket = async ( file: ProcessedFile, diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 33446e4e08..6de421ca9b 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -11,9 +11,8 @@ import { const TYPE_JSON = "json"; -export function hasFileHash(file: Metadata) { - return file.hash || (file.imageHash && file.videoHash); -} +export const hasFileHash = (file: Metadata) => + file.hash || (file.imageHash && file.videoHash); export function segregateMetadataAndMediaFiles( filesWithCollectionToUpload: FileWithCollection[], From 48bace50df5bd9ee0eedf50d1115fb33827457db Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:04:29 +0530 Subject: [PATCH 56/82] Extract --- .../src/services/upload/metadataService.ts | 159 +----------------- .../photos/src/services/upload/takeout.ts | 155 +++++++++++++++++ .../src/services/upload/uploadManager.ts | 5 +- .../src/services/upload/uploadService.ts | 4 +- web/apps/photos/src/types/upload/index.ts | 9 - web/apps/photos/tests/upload.test.ts | 2 +- 6 files changed, 169 insertions(+), 165 deletions(-) create mode 100644 web/apps/photos/src/services/upload/takeout.ts diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index f590e50a36..cb17ba4c16 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,4 +1,3 @@ -import { ensureElectron } from "@/next/electron"; import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; @@ -19,11 +18,8 @@ import { FilePublicMagicMetadataProps } from "types/file"; import { FileTypeInfo, LivePhotoAssets, - Location, Metadata, ParsedExtractedMetadata, - ParsedMetadataJSON, - ParsedMetadataJSONMap, type DataStream, type FileWithCollection, type FileWithCollection2, @@ -32,15 +28,15 @@ import { } from "types/upload"; import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; +import { + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + getClippedMetadataJSONMapKeyForFile, + getMetadataJSONMapKeyForFile, + type ParsedMetadataJSON, +} from "./takeout"; import uploadCancelService from "./uploadCancelService"; import { getFileName } from "./uploadService"; -const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { - creationTime: null, - modificationTime: null, - ...NULL_LOCATION, -}; - const EXIF_TAGS_NEEDED = [ "DateTimeOriginal", "CreateDate", @@ -59,8 +55,6 @@ const EXIF_TAGS_NEEDED = [ "MetadataDate", ]; -export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; - export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { location: NULL_LOCATION, creationTime: null, @@ -138,115 +132,6 @@ export async function getImageMetadata( return imageMetadata; } -export const getMetadataJSONMapKeyForJSON = ( - collectionID: number, - jsonFileName: string, -) => { - let title = jsonFileName.slice(0, -1 * ".json".length); - const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); - if (endsWithNumberedSuffixWithBrackets) { - title = title.slice( - 0, - -1 * endsWithNumberedSuffixWithBrackets[0].length, - ); - const [name, extension] = splitFilenameAndExtension(title); - return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; - } - return `${collectionID}-${title}`; -}; - -// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name -// so we need to use the clipped file name to get the metadataJSON file -export const getClippedMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${fileName.slice( - 0, - MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, - )}`; -}; - -export const getMetadataJSONMapKeyForFile = ( - collectionID: number, - fileName: string, -) => { - return `${collectionID}-${getFileOriginalName(fileName)}`; -}; - -export async function parseMetadataJSON( - receivedFile: File | ElectronFile | string, -) { - try { - let text: string; - if (typeof receivedFile == "string") { - text = await ensureElectron().fs.readTextFile(receivedFile); - } else { - if (!(receivedFile instanceof File)) { - receivedFile = new File( - [await receivedFile.blob()], - receivedFile.name, - ); - } - text = await receivedFile.text(); - } - - return parseMetadataJSONText(text); - } catch (e) { - log.error("parseMetadataJSON failed", e); - // ignore - } -} - -export async function parseMetadataJSONText(text: string) { - const metadataJSON: object = JSON.parse(text); - - const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; - if (!metadataJSON) { - return; - } - - if ( - metadataJSON["photoTakenTime"] && - metadataJSON["photoTakenTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["photoTakenTime"]["timestamp"] * 1000000; - } else if ( - metadataJSON["creationTime"] && - metadataJSON["creationTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["creationTime"]["timestamp"] * 1000000; - } - if ( - metadataJSON["modificationTime"] && - metadataJSON["modificationTime"]["timestamp"] - ) { - parsedMetadataJSON.modificationTime = - metadataJSON["modificationTime"]["timestamp"] * 1000000; - } - let locationData: Location = NULL_LOCATION; - if ( - metadataJSON["geoData"] && - (metadataJSON["geoData"]["latitude"] !== 0.0 || - metadataJSON["geoData"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoData"]; - } else if ( - metadataJSON["geoDataExif"] && - (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || - metadataJSON["geoDataExif"]["longitude"] !== 0.0) - ) { - locationData = metadataJSON["geoDataExif"]; - } - if (locationData !== null) { - parsedMetadataJSON.latitude = locationData.latitude; - parsedMetadataJSON.longitude = locationData.longitude; - } - return parsedMetadataJSON; -} - // tries to extract date from file name if available else returns null export function extractDateFromFileName(filename: string): number { try { @@ -283,32 +168,6 @@ function convertSignalNameToFusedDateString(filename: string) { return `${dateStringParts[1]}${dateStringParts[2]}${dateStringParts[3]}-${dateStringParts[4]}`; } -const EDITED_FILE_SUFFIX = "-edited"; - -/* - Get the original file name for edited file to associate it to original file's metadataJSON file - as edited file doesn't have their own metadata file -*/ -function getFileOriginalName(fileName: string) { - let originalName: string = null; - const [nameWithoutExtension, extension] = - splitFilenameAndExtension(fileName); - - const isEditedFile = nameWithoutExtension.endsWith(EDITED_FILE_SUFFIX); - if (isEditedFile) { - originalName = nameWithoutExtension.slice( - 0, - -1 * EDITED_FILE_SUFFIX.length, - ); - } else { - originalName = nameWithoutExtension; - } - if (extension) { - originalName += "." + extension; - } - return originalName; -} - async function getVideoMetadata(file: File | ElectronFile) { let videoMetadata = NULL_EXTRACTED_METADATA; try { @@ -356,7 +215,7 @@ export async function getLivePhotoFileType( export const extractAssetMetadata = async ( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, collectionID: number, fileTypeInfo: FileTypeInfo, @@ -380,7 +239,7 @@ export const extractAssetMetadata = async ( async function extractFileMetadata( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, collectionID: number, fileTypeInfo: FileTypeInfo, rawFile: File | ElectronFile | string, @@ -412,7 +271,7 @@ async function extractFileMetadata( async function extractLivePhotoMetadata( worker: Remote, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, collectionID: number, fileTypeInfo: FileTypeInfo, livePhotoAssets: LivePhotoAssets2, diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts new file mode 100644 index 0000000000..92849cac28 --- /dev/null +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -0,0 +1,155 @@ +/** @file Dealing with the JSON metadata in Google Takeouts */ + +import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; +import log from "@/next/log"; +import type { ElectronFile } from "@/next/types/file"; +import { NULL_LOCATION } from "constants/upload"; +import { type Location } from "types/upload"; + +export interface ParsedMetadataJSON { + creationTime: number; + modificationTime: number; + latitude: number; + longitude: number; +} + +export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; + +export const getMetadataJSONMapKeyForJSON = ( + collectionID: number, + jsonFileName: string, +) => { + let title = jsonFileName.slice(0, -1 * ".json".length); + const endsWithNumberedSuffixWithBrackets = title.match(/\(\d+\)$/); + if (endsWithNumberedSuffixWithBrackets) { + title = title.slice( + 0, + -1 * endsWithNumberedSuffixWithBrackets[0].length, + ); + const [name, extension] = nameAndExtension(title); + return `${collectionID}-${name}${endsWithNumberedSuffixWithBrackets[0]}.${extension}`; + } + return `${collectionID}-${title}`; +}; + +// if the file name is greater than MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT(46) , then google photos clips the file name +// so we need to use the clipped file name to get the metadataJSON file +export const getClippedMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${fileName.slice( + 0, + MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, + )}`; +}; + +export const getMetadataJSONMapKeyForFile = ( + collectionID: number, + fileName: string, +) => { + return `${collectionID}-${getFileOriginalName(fileName)}`; +}; + +const EDITED_FILE_SUFFIX = "-edited"; + +/* + Get the original file name for edited file to associate it to original file's metadataJSON file + as edited file doesn't have their own metadata file +*/ +function getFileOriginalName(fileName: string) { + let originalName: string = null; + const [name, extension] = nameAndExtension(fileName); + + const isEditedFile = name.endsWith(EDITED_FILE_SUFFIX); + if (isEditedFile) { + originalName = name.slice(0, -1 * EDITED_FILE_SUFFIX.length); + } else { + originalName = name; + } + if (extension) { + originalName += "." + extension; + } + return originalName; +} + +export async function parseMetadataJSON( + receivedFile: File | ElectronFile | string, +) { + try { + let text: string; + if (typeof receivedFile == "string") { + text = await ensureElectron().fs.readTextFile(receivedFile); + } else { + if (!(receivedFile instanceof File)) { + receivedFile = new File( + [await receivedFile.blob()], + receivedFile.name, + ); + } + text = await receivedFile.text(); + } + + return parseMetadataJSONText(text); + } catch (e) { + log.error("parseMetadataJSON failed", e); + // ignore + } +} + +const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { + creationTime: null, + modificationTime: null, + latitude: null, longitude: null + ...NULL_LOCATION, +}; + +export async function parseMetadataJSONText(text: string) { + const metadataJSON: object = JSON.parse(text); + + const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; + if (!metadataJSON) { + return; + } + + if ( + metadataJSON["photoTakenTime"] && + metadataJSON["photoTakenTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["photoTakenTime"]["timestamp"] * 1000000; + } else if ( + metadataJSON["creationTime"] && + metadataJSON["creationTime"]["timestamp"] + ) { + parsedMetadataJSON.creationTime = + metadataJSON["creationTime"]["timestamp"] * 1000000; + } + if ( + metadataJSON["modificationTime"] && + metadataJSON["modificationTime"]["timestamp"] + ) { + parsedMetadataJSON.modificationTime = + metadataJSON["modificationTime"]["timestamp"] * 1000000; + } + let locationData: Location = NULL_LOCATION; + if ( + metadataJSON["geoData"] && + (metadataJSON["geoData"]["latitude"] !== 0.0 || + metadataJSON["geoData"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoData"]; + } else if ( + metadataJSON["geoDataExif"] && + (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || + metadataJSON["geoDataExif"]["longitude"] !== 0.0) + ) { + locationData = metadataJSON["geoDataExif"]; + } + if (locationData !== null) { + parsedMetadataJSON.latitude = locationData.latitude; + parsedMetadataJSON.longitude = locationData.longitude; + } + return parsedMetadataJSON; +} diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index f2a3861924..bb6c5aba88 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -26,8 +26,6 @@ import { EncryptedEnteFile, EnteFile } from "types/file"; import { SetFiles } from "types/gallery"; import { FileWithCollection, - ParsedMetadataJSON, - ParsedMetadataJSONMap, PublicUploadProps, type FileWithCollection2, } from "types/upload"; @@ -50,6 +48,7 @@ import { getMetadataJSONMapKeyForJSON, parseMetadataJSON, } from "./metadataService"; +import type { ParsedMetadataJSON } from "./takeout"; import uploadCancelService from "./uploadCancelService"; import UploadService, { assetName, @@ -264,7 +263,7 @@ class UploadManager { private cryptoWorkers = new Array< ComlinkWorker >(MAX_CONCURRENT_UPLOADS); - private parsedMetadataJSONMap: ParsedMetadataJSONMap; + private parsedMetadataJSONMap: Map; private filesToBeUploaded: FileWithCollection2[]; private remainingFiles: FileWithCollection2[] = []; private failedFiles: FileWithCollection2[]; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 442f6e6cdf..f1583dabff 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -29,7 +29,6 @@ import { FileInMemory, FileTypeInfo, FileWithMetadata, - ParsedMetadataJSONMap, ProcessedFile, PublicUploadProps, UploadAsset, @@ -64,6 +63,7 @@ import { } from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; +import type { ParsedMetadataJSON } from "./takeout"; /** Upload files to cloud storage */ class UploadService { @@ -169,7 +169,7 @@ export const uploader = async ( worker: Remote, existingFiles: EnteFile[], fileWithCollection: FileWithCollection2, - parsedMetadataJSONMap: ParsedMetadataJSONMap, + parsedMetadataJSONMap: Map, uploaderName: string, isCFUploadProxyDisabled: boolean, makeProgessTracker: MakeProgressTracker, diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 98e129bf4f..95913531f3 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -38,13 +38,6 @@ export interface Location { longitude: number; } -export interface ParsedMetadataJSON { - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; -} - export interface MultipartUploadURLs { objectKey: string; partURLs: string[]; @@ -93,8 +86,6 @@ export interface FileWithCollection2 extends UploadAsset2 { collectionID?: number; } -export type ParsedMetadataJSONMap = Map; - export interface UploadURL { url: string; objectKey: string; diff --git a/web/apps/photos/tests/upload.test.ts b/web/apps/photos/tests/upload.test.ts index 6e58cf0c2d..5a05bb991d 100644 --- a/web/apps/photos/tests/upload.test.ts +++ b/web/apps/photos/tests/upload.test.ts @@ -7,7 +7,7 @@ import { getClippedMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForFile, getMetadataJSONMapKeyForJSON, -} from "services/upload/metadataService"; +} from "services/upload/takeout"; import { getUserDetailsV2 } from "services/userService"; import { groupFilesBasedOnCollectionID } from "utils/file"; From 9103dadc6f0a33b3c47311a28c3f9b2e3a64d6bb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:13:03 +0530 Subject: [PATCH 57/82] Tinker --- web/apps/photos/src/services/upload/takeout.ts | 17 +++++++++-------- .../photos/src/services/upload/uploadManager.ts | 15 ++++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 92849cac28..660b6d0e71 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -74,9 +74,10 @@ function getFileOriginalName(fileName: string) { return originalName; } -export async function parseMetadataJSON( +/** Try to parse the contents of a metadata JSON file in a Google Takeout. */ +export const tryParseTakeoutMetadataJSON = async ( receivedFile: File | ElectronFile | string, -) { +): Promise => { try { let text: string; if (typeof receivedFile == "string") { @@ -93,8 +94,8 @@ export async function parseMetadataJSON( return parseMetadataJSONText(text); } catch (e) { - log.error("parseMetadataJSON failed", e); - // ignore + log.error("Failed to parse takeout metadata JSON", e); + return undefined; } } @@ -105,13 +106,13 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { ...NULL_LOCATION, }; -export async function parseMetadataJSONText(text: string) { +const parseMetadataJSONText = (text: string) => { const metadataJSON: object = JSON.parse(text); + if (!metadataJSON) { + return undefined; + } const parsedMetadataJSON: ParsedMetadataJSON = NULL_PARSED_METADATA_JSON; - if (!metadataJSON) { - return; - } if ( metadataJSON["photoTakenTime"] && diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index bb6c5aba88..c7b6ca942d 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -43,12 +43,12 @@ import { segregateMetadataAndMediaFiles2, } from "utils/upload"; import { getLocalFiles } from "../fileService"; +import { clusterLivePhotoFiles } from "./metadataService"; import { - clusterLivePhotoFiles, getMetadataJSONMapKeyForJSON, - parseMetadataJSON, -} from "./metadataService"; -import type { ParsedMetadataJSON } from "./takeout"; + tryParseTakeoutMetadataJSON, + type ParsedMetadataJSON, +} from "./takeout"; import uploadCancelService from "./uploadCancelService"; import UploadService, { assetName, @@ -523,11 +523,12 @@ class UploadManager { log.info(`parsing metadata json file ${name}`); - const parsedMetadataJSON = await parseMetadataJSON(file); - if (parsedMetadataJSON) { + const metadataJSON = + await tryParseTakeoutMetadataJSON(file); + if (metadataJSON) { this.parsedMetadataJSONMap.set( getMetadataJSONMapKeyForJSON(collectionID, name), - parsedMetadataJSON && { ...parsedMetadataJSON }, + metadataJSON && { ...metadataJSON }, ); this.uiService.increaseFileUploaded(); } From 2bee444078a9f6a45a183ef039c43e7a782240b3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:19:39 +0530 Subject: [PATCH 58/82] Tinker --- .../src/services/upload/metadataService.ts | 150 ++++++++---------- .../src/services/upload/uploadManager.ts | 18 +-- 2 files changed, 72 insertions(+), 96 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index cb17ba4c16..c3a9ca252a 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,4 +1,4 @@ -import { getFileNameSize } from "@/next/file"; +import { getFileNameSize, nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; @@ -188,13 +188,6 @@ async function getVideoMetadata(file: File | ElectronFile) { return videoMetadata; } -interface LivePhotoIdentifier { - collectionID: number; - fileType: FILE_TYPE; - name: string; - size: number; -} - const UNDERSCORE_THREE = "_3"; // Note: The icloud-photos-downloader library appends _HVEC to the end of the filename in case of live photos // https://github.com/icloud-photos-downloader/icloud_photos_downloader @@ -312,74 +305,64 @@ export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { return livePhotoAssets.image.size + livePhotoAssets.video.size; } -export async function clusterLivePhotoFiles(mediaFiles: FileWithCollection2[]) { +/** + * Go through the given files, combining any sibling image + video assets into a + * single live photo when appropriate. + */ +export const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { try { - const analysedMediaFiles: FileWithCollection2[] = []; + const result: FileWithCollection2[] = []; mediaFiles - .sort((firstMediaFile, secondMediaFile) => - splitFilenameAndExtension( - getFileName(firstMediaFile.file), - )[0].localeCompare( - splitFilenameAndExtension( - getFileName(secondMediaFile.file), - )[0], + .sort((f, g) => + nameAndExtension(getFileName(f.file))[0].localeCompare( + nameAndExtension(getFileName(g.file))[0], ), ) - .sort( - (firstMediaFile, secondMediaFile) => - firstMediaFile.collectionID - secondMediaFile.collectionID, - ); + .sort((f, g) => f.collectionID - g.collectionID); let index = 0; while (index < mediaFiles.length - 1) { if (uploadCancelService.isUploadCancelationRequested()) { throw Error(CustomError.UPLOAD_CANCELLED); } - const firstMediaFile = mediaFiles[index]; - const secondMediaFile = mediaFiles[index + 1]; - const firstFileType = - getFileTypeFromExtensionForLivePhotoClustering( - getFileName(firstMediaFile.file), - ); - const secondFileType = - getFileTypeFromExtensionForLivePhotoClustering( - getFileName(secondMediaFile.file), - ); - const firstFileIdentifier: LivePhotoIdentifier = { - collectionID: firstMediaFile.collectionID, - fileType: firstFileType, - name: getFileName(firstMediaFile.file), + const f = mediaFiles[index]; + const g = mediaFiles[index + 1]; + const fFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(f.file), + ); + const gFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(g.file), + ); + const fFileIdentifier: LivePhotoIdentifier = { + collectionID: f.collectionID, + fileType: fFileType, + name: getFileName(f.file), /* TODO(MR): ElectronFile changes */ - size: (firstMediaFile as FileWithCollection).file.size, + size: (f as FileWithCollection).file.size, }; - const secondFileIdentifier: LivePhotoIdentifier = { - collectionID: secondMediaFile.collectionID, - fileType: secondFileType, - name: getFileName(secondMediaFile.file), + const gFileIdentifier: LivePhotoIdentifier = { + collectionID: g.collectionID, + fileType: gFileType, + name: getFileName(g.file), /* TODO(MR): ElectronFile changes */ - size: (secondMediaFile as FileWithCollection).file.size, + size: (g as FileWithCollection).file.size, }; - if ( - areFilesLivePhotoAssets( - firstFileIdentifier, - secondFileIdentifier, - ) - ) { + if (areLivePhotoAssets(fFileIdentifier, gFileIdentifier)) { let imageFile: File | ElectronFile | string; let videoFile: File | ElectronFile | string; if ( - firstFileType === FILE_TYPE.IMAGE && - secondFileType === FILE_TYPE.VIDEO + fFileType === FILE_TYPE.IMAGE && + gFileType === FILE_TYPE.VIDEO ) { - imageFile = firstMediaFile.file; - videoFile = secondMediaFile.file; + imageFile = f.file; + videoFile = g.file; } else { - videoFile = firstMediaFile.file; - imageFile = secondMediaFile.file; + videoFile = f.file; + imageFile = g.file; } - const livePhotoLocalID = firstMediaFile.localID; - analysedMediaFiles.push({ + const livePhotoLocalID = f.localID; + result.push({ localID: livePhotoLocalID, - collectionID: firstMediaFile.collectionID, + collectionID: f.collectionID, isLivePhoto: true, livePhotoAssets: { image: imageFile, @@ -388,20 +371,20 @@ export async function clusterLivePhotoFiles(mediaFiles: FileWithCollection2[]) { }); index += 2; } else { - analysedMediaFiles.push({ - ...firstMediaFile, + result.push({ + ...f, isLivePhoto: false, }); index += 1; } } if (index === mediaFiles.length - 1) { - analysedMediaFiles.push({ + result.push({ ...mediaFiles[index], isLivePhoto: false, }); } - return analysedMediaFiles; + return result; } catch (e) { if (e.message === CustomError.UPLOAD_CANCELLED) { throw e; @@ -410,42 +393,44 @@ export async function clusterLivePhotoFiles(mediaFiles: FileWithCollection2[]) { throw e; } } +}; + +interface LivePhotoIdentifier { + collectionID: number; + fileType: FILE_TYPE; + name: string; + size: number; } -function areFilesLivePhotoAssets( - firstFileIdentifier: LivePhotoIdentifier, - secondFileIdentifier: LivePhotoIdentifier, -) { - const haveSameCollectionID = - firstFileIdentifier.collectionID === secondFileIdentifier.collectionID; - const areNotSameFileType = - firstFileIdentifier.fileType !== secondFileIdentifier.fileType; +const areLivePhotoAssets = (f: LivePhotoIdentifier, g: LivePhotoIdentifier) => { + const haveSameCollectionID = f.collectionID === g.collectionID; + const areNotSameFileType = f.fileType !== g.fileType; let firstFileNameWithoutSuffix: string; let secondFileNameWithoutSuffix: string; - if (firstFileIdentifier.fileType === FILE_TYPE.IMAGE) { + if (f.fileType === FILE_TYPE.IMAGE) { firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(firstFileIdentifier.name), + getFileNameWithoutExtension(f.name), // Note: The Google Live Photo image file can have video extension appended as suffix, passing that to removePotentialLivePhotoSuffix to remove it // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) - getFileExtensionWithDot(secondFileIdentifier.name), + getFileExtensionWithDot(g.name), ); secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(secondFileIdentifier.name), + getFileNameWithoutExtension(g.name), ); } else { firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(firstFileIdentifier.name), + getFileNameWithoutExtension(f.name), ); secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(secondFileIdentifier.name), - getFileExtensionWithDot(firstFileIdentifier.name), + getFileNameWithoutExtension(g.name), + getFileExtensionWithDot(f.name), ); } if ( haveSameCollectionID && - isImageOrVideo(firstFileIdentifier.fileType) && - isImageOrVideo(secondFileIdentifier.fileType) && + isImageOrVideo(f.fileType) && + isImageOrVideo(g.fileType) && areNotSameFileType && firstFileNameWithoutSuffix === secondFileNameWithoutSuffix ) { @@ -455,23 +440,20 @@ function areFilesLivePhotoAssets( // I did that based on the assumption that live photo assets ideally would not be larger than LIVE_PHOTO_ASSET_SIZE_LIMIT // also zipping library doesn't support stream as a input if ( - firstFileIdentifier.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT && - secondFileIdentifier.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT + f.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT && + g.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT ) { return true; } else { log.error( `${CustomError.TOO_LARGE_LIVE_PHOTO_ASSETS} - ${JSON.stringify({ - fileSizes: [ - firstFileIdentifier.size, - secondFileIdentifier.size, - ], + fileSizes: [f.size, g.size], })}`, ); } } return false; -} +}; function removePotentialLivePhotoSuffix( filenameWithoutExtension: string, diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index c7b6ca942d..0cc9db38fc 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -367,19 +367,13 @@ class UploadManager { ); await this.parseMetadataJSONFiles(metadataJSONFiles); } + if (mediaFiles.length) { - log.info(`clusterLivePhotoFiles started`); - const analysedMediaFiles = - await clusterLivePhotoFiles(mediaFiles); - log.info(`clusterLivePhotoFiles ended`); - log.info( - `got live photos: ${ - mediaFiles.length !== analysedMediaFiles.length - }`, - ); + const clusteredMediaFiles = await clusterLivePhotos(mediaFiles); + this.uiService.setFilenames( new Map( - analysedMediaFiles.map((mediaFile) => [ + clusteredMediaFiles.map((mediaFile) => [ mediaFile.localID, assetName(mediaFile), ]), @@ -387,10 +381,10 @@ class UploadManager { ); this.uiService.setHasLivePhoto( - mediaFiles.length !== analysedMediaFiles.length, + mediaFiles.length !== clusteredMediaFiles.length, ); - await this.uploadMediaFiles(analysedMediaFiles); + await this.uploadMediaFiles(clusteredMediaFiles); } } catch (e) { if (e.message === CustomError.UPLOAD_CANCELLED) { From 88c2a52edff644c7641d2b861380fce65be021b4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:37:51 +0530 Subject: [PATCH 59/82] Spruce --- .../src/services/upload/metadataService.ts | 141 ++++++++---------- .../photos/src/services/upload/takeout.ts | 5 +- .../src/services/upload/uploadManager.ts | 8 +- 3 files changed, 72 insertions(+), 82 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index c3a9ca252a..a9d5fec4c8 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -34,7 +34,6 @@ import { getMetadataJSONMapKeyForFile, type ParsedMetadataJSON, } from "./takeout"; -import uploadCancelService from "./uploadCancelService"; import { getFileName } from "./uploadService"; const EXIF_TAGS_NEEDED = [ @@ -310,89 +309,77 @@ export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { * single live photo when appropriate. */ export const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { - try { - const result: FileWithCollection2[] = []; - mediaFiles - .sort((f, g) => - nameAndExtension(getFileName(f.file))[0].localeCompare( - nameAndExtension(getFileName(g.file))[0], - ), - ) - .sort((f, g) => f.collectionID - g.collectionID); - let index = 0; - while (index < mediaFiles.length - 1) { - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } - const f = mediaFiles[index]; - const g = mediaFiles[index + 1]; - const fFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(f.file), - ); - const gFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(g.file), - ); - const fFileIdentifier: LivePhotoIdentifier = { - collectionID: f.collectionID, - fileType: fFileType, - name: getFileName(f.file), - /* TODO(MR): ElectronFile changes */ - size: (f as FileWithCollection).file.size, - }; - const gFileIdentifier: LivePhotoIdentifier = { - collectionID: g.collectionID, - fileType: gFileType, - name: getFileName(g.file), - /* TODO(MR): ElectronFile changes */ - size: (g as FileWithCollection).file.size, - }; - if (areLivePhotoAssets(fFileIdentifier, gFileIdentifier)) { - let imageFile: File | ElectronFile | string; - let videoFile: File | ElectronFile | string; - if ( - fFileType === FILE_TYPE.IMAGE && - gFileType === FILE_TYPE.VIDEO - ) { - imageFile = f.file; - videoFile = g.file; - } else { - videoFile = f.file; - imageFile = g.file; - } - const livePhotoLocalID = f.localID; - result.push({ - localID: livePhotoLocalID, - collectionID: f.collectionID, - isLivePhoto: true, - livePhotoAssets: { - image: imageFile, - video: videoFile, - }, - }); - index += 2; + const result: FileWithCollection2[] = []; + mediaFiles + .sort((f, g) => + nameAndExtension(getFileName(f.file))[0].localeCompare( + nameAndExtension(getFileName(g.file))[0], + ), + ) + .sort((f, g) => f.collectionID - g.collectionID); + let index = 0; + while (index < mediaFiles.length - 1) { + const f = mediaFiles[index]; + const g = mediaFiles[index + 1]; + const fFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(f.file), + ); + const gFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(g.file), + ); + const fFileIdentifier: LivePhotoIdentifier = { + collectionID: f.collectionID, + fileType: fFileType, + name: getFileName(f.file), + /* TODO(MR): ElectronFile changes */ + size: (f as FileWithCollection).file.size, + }; + const gFileIdentifier: LivePhotoIdentifier = { + collectionID: g.collectionID, + fileType: gFileType, + name: getFileName(g.file), + /* TODO(MR): ElectronFile changes */ + size: (g as FileWithCollection).file.size, + }; + if (areLivePhotoAssets(fFileIdentifier, gFileIdentifier)) { + let imageFile: File | ElectronFile | string; + let videoFile: File | ElectronFile | string; + if ( + fFileType === FILE_TYPE.IMAGE && + gFileType === FILE_TYPE.VIDEO + ) { + imageFile = f.file; + videoFile = g.file; } else { - result.push({ - ...f, - isLivePhoto: false, - }); - index += 1; + videoFile = f.file; + imageFile = g.file; } - } - if (index === mediaFiles.length - 1) { + const livePhotoLocalID = f.localID; result.push({ - ...mediaFiles[index], + localID: livePhotoLocalID, + collectionID: f.collectionID, + isLivePhoto: true, + livePhotoAssets: { + image: imageFile, + video: videoFile, + }, + }); + index += 2; + } else { + result.push({ + ...f, isLivePhoto: false, }); - } - return result; - } catch (e) { - if (e.message === CustomError.UPLOAD_CANCELLED) { - throw e; - } else { - log.error("failed to cluster live photo", e); - throw e; + index += 1; } } + if (index === mediaFiles.length - 1) { + result.push({ + ...mediaFiles[index], + isLivePhoto: false, + }); + } + return result; }; interface LivePhotoIdentifier { diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 660b6d0e71..ba6f402b43 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -97,12 +97,11 @@ export const tryParseTakeoutMetadataJSON = async ( log.error("Failed to parse takeout metadata JSON", e); return undefined; } -} +}; const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { creationTime: null, modificationTime: null, - latitude: null, longitude: null ...NULL_LOCATION, }; @@ -153,4 +152,4 @@ const parseMetadataJSONText = (text: string) => { parsedMetadataJSON.longitude = locationData.longitude; } return parsedMetadataJSON; -} +}; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 0cc9db38fc..04080c62c9 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -43,7 +43,7 @@ import { segregateMetadataAndMediaFiles2, } from "utils/upload"; import { getLocalFiles } from "../fileService"; -import { clusterLivePhotoFiles } from "./metadataService"; +import { clusterLivePhotoFiles, clusterLivePhotos } from "./metadataService"; import { getMetadataJSONMapKeyForJSON, tryParseTakeoutMetadataJSON, @@ -369,7 +369,11 @@ class UploadManager { } if (mediaFiles.length) { - const clusteredMediaFiles = await clusterLivePhotos(mediaFiles); + const clusteredMediaFiles = clusterLivePhotos(mediaFiles); + + if (uploadCancelService.isUploadCancelationRequested()) { + throw Error(CustomError.UPLOAD_CANCELLED); + } this.uiService.setFilenames( new Map( From 27185c333c687c37a67aed3d8e7bbadfb9526116 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 10:58:04 +0530 Subject: [PATCH 60/82] Prune --- .../src/services/upload/metadataService.ts | 56 +++++++------------ 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index a9d5fec4c8..fa744bb853 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -187,11 +187,6 @@ async function getVideoMetadata(file: File | ElectronFile) { return videoMetadata; } -const UNDERSCORE_THREE = "_3"; -// Note: The icloud-photos-downloader library appends _HVEC to the end of the filename in case of live photos -// https://github.com/icloud-photos-downloader/icloud_photos_downloader -const UNDERSCORE_HEVC = "_HVEC"; - export async function getLivePhotoFileType( livePhotoAssets: LivePhotoAssets, ): Promise { @@ -442,32 +437,31 @@ const areLivePhotoAssets = (f: LivePhotoIdentifier, g: LivePhotoIdentifier) => { return false; }; -function removePotentialLivePhotoSuffix( - filenameWithoutExtension: string, - suffix?: string, -) { - let presentSuffix: string; - if (filenameWithoutExtension.endsWith(UNDERSCORE_THREE)) { - presentSuffix = UNDERSCORE_THREE; - } else if (filenameWithoutExtension.endsWith(UNDERSCORE_HEVC)) { - presentSuffix = UNDERSCORE_HEVC; +const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { + const suffix_3 = "_3"; + + // The icloud-photos-downloader library appends _HVEC to the end of the + // filename in case of live photos. + // + // https://github.com/icloud-photos-downloader/icloud_photos_downloader + const suffix_hvec = "_HVEC"; + + let foundSuffix: string | undefined; + if (name.endsWith(suffix_3)) { + foundSuffix = suffix_3; } else if ( - filenameWithoutExtension.endsWith(UNDERSCORE_HEVC.toLowerCase()) + name.endsWith(suffix_hvec) || + name.endsWith(suffix_hvec.toLowerCase()) ) { - presentSuffix = UNDERSCORE_HEVC.toLowerCase(); + foundSuffix = suffix_hvec; } else if (suffix) { - if (filenameWithoutExtension.endsWith(suffix)) { - presentSuffix = suffix; - } else if (filenameWithoutExtension.endsWith(suffix.toLowerCase())) { - presentSuffix = suffix.toLowerCase(); + if (name.endsWith(suffix) || name.endsWith(suffix.toLowerCase())) { + foundSuffix = suffix; } } - if (presentSuffix) { - return filenameWithoutExtension.slice(0, presentSuffix.length * -1); - } else { - return filenameWithoutExtension; - } -} + + return foundSuffix ? name.slice(0, foundSuffix.length * -1) : name; +}; function getFileNameWithoutExtension(filename: string) { const lastDotPosition = filename.lastIndexOf("."); @@ -481,16 +475,6 @@ function getFileExtensionWithDot(filename: string) { else return filename.slice(lastDotPosition); } -function splitFilenameAndExtension(filename: string): [string, string] { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return [filename, null]; - else - return [ - filename.slice(0, lastDotPosition), - filename.slice(lastDotPosition + 1), - ]; -} - const isImageOrVideo = (fileType: FILE_TYPE) => [FILE_TYPE.IMAGE, FILE_TYPE.VIDEO].includes(fileType); From 5b928883a6a599df3dab7910413a8c36a0fc6efb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:03:49 +0530 Subject: [PATCH 61/82] Tinker --- .../src/services/upload/metadataService.ts | 21 ++++++++----------- web/packages/shared/error/index.ts | 1 - 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index fa744bb853..a981f99bc6 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -416,21 +416,18 @@ const areLivePhotoAssets = (f: LivePhotoIdentifier, g: LivePhotoIdentifier) => { areNotSameFileType && firstFileNameWithoutSuffix === secondFileNameWithoutSuffix ) { - const LIVE_PHOTO_ASSET_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB + // Also check that the size of an individual Live Photo asset is less + // than an (arbitrary) limit. This should be true in practice as the + // videos for a live photo are a few seconds long. Further on, the + // zipping library that we use doesn't support stream as a input. - // checks size of live Photo assets are less than allowed limit - // I did that based on the assumption that live photo assets ideally would not be larger than LIVE_PHOTO_ASSET_SIZE_LIMIT - // also zipping library doesn't support stream as a input - if ( - f.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT && - g.size <= LIVE_PHOTO_ASSET_SIZE_LIMIT - ) { + const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ + + if (f.size <= maxAssetSize && g.size <= maxAssetSize) { return true; } else { - log.error( - `${CustomError.TOO_LARGE_LIVE_PHOTO_ASSETS} - ${JSON.stringify({ - fileSizes: [f.size, g.size], - })}`, + log.info( + `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, ); } } diff --git a/web/packages/shared/error/index.ts b/web/packages/shared/error/index.ts index fab3161b21..cf01a38435 100644 --- a/web/packages/shared/error/index.ts +++ b/web/packages/shared/error/index.ts @@ -48,7 +48,6 @@ export const CustomError = { SUBSCRIPTION_NEEDED: "subscription not present", NOT_FOUND: "not found ", NO_METADATA: "no metadata", - TOO_LARGE_LIVE_PHOTO_ASSETS: "too large live photo assets", NOT_A_DATE: "not a date", NOT_A_LOCATION: "not a location", FILE_ID_NOT_FOUND: "file with id not found", From bded3c6706cb3dd88e24a0f39581931c4a091b1b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:14:34 +0530 Subject: [PATCH 62/82] Prune --- .../src/services/upload/metadataService.ts | 107 ++++++++---------- 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index a981f99bc6..22dc7d61f4 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -322,21 +322,21 @@ export const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { const gFileType = getFileTypeFromExtensionForLivePhotoClustering( getFileName(g.file), ); - const fFileIdentifier: LivePhotoIdentifier = { + const fa: PotentialLivePhotoAsset = { collectionID: f.collectionID, fileType: fFileType, - name: getFileName(f.file), + fileName: getFileName(f.file), /* TODO(MR): ElectronFile changes */ size: (f as FileWithCollection).file.size, }; - const gFileIdentifier: LivePhotoIdentifier = { + const ga: PotentialLivePhotoAsset = { collectionID: g.collectionID, fileType: gFileType, - name: getFileName(g.file), + fileName: getFileName(g.file), /* TODO(MR): ElectronFile changes */ size: (g as FileWithCollection).file.size, }; - if (areLivePhotoAssets(fFileIdentifier, gFileIdentifier)) { + if (areLivePhotoAssets(fa, ga)) { let imageFile: File | ElectronFile | string; let videoFile: File | ElectronFile | string; if ( @@ -377,61 +377,61 @@ export const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { return result; }; -interface LivePhotoIdentifier { +interface PotentialLivePhotoAsset { collectionID: number; fileType: FILE_TYPE; - name: string; + fileName: string; size: number; } -const areLivePhotoAssets = (f: LivePhotoIdentifier, g: LivePhotoIdentifier) => { - const haveSameCollectionID = f.collectionID === g.collectionID; - const areNotSameFileType = f.fileType !== g.fileType; +const areLivePhotoAssets = ( + f: PotentialLivePhotoAsset, + g: PotentialLivePhotoAsset, +) => { + if (f.collectionID != g.collectionID) return false; - let firstFileNameWithoutSuffix: string; - let secondFileNameWithoutSuffix: string; - if (f.fileType === FILE_TYPE.IMAGE) { - firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(f.name), - // Note: The Google Live Photo image file can have video extension appended as suffix, passing that to removePotentialLivePhotoSuffix to remove it + const [fName, fExt] = nameAndExtension(f.fileName); + const [gName, gExt] = nameAndExtension(g.fileName); + + let fPrunedName: string; + let gPrunedName: string; + if (f.fileType == FILE_TYPE.IMAGE && g.fileType == FILE_TYPE.VIDEO) { + fPrunedName = removePotentialLivePhotoSuffix( + fName, + // A Google Live Photo image file can have video extension appended + // as suffix, so we pass that to removePotentialLivePhotoSuffix to + // remove it. + // // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) - getFileExtensionWithDot(g.name), + gExt ? `.${gExt}` : undefined, ); - secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(g.name), + gPrunedName = removePotentialLivePhotoSuffix(gName); + } else if (f.fileType == FILE_TYPE.VIDEO && g.fileType == FILE_TYPE.IMAGE) { + fPrunedName = removePotentialLivePhotoSuffix(fName); + gPrunedName = removePotentialLivePhotoSuffix( + gName, + fExt ? `.${fExt}` : undefined, ); } else { - firstFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(f.name), - ); - secondFileNameWithoutSuffix = removePotentialLivePhotoSuffix( - getFileNameWithoutExtension(g.name), - getFileExtensionWithDot(f.name), - ); + return false; } - if ( - haveSameCollectionID && - isImageOrVideo(f.fileType) && - isImageOrVideo(g.fileType) && - areNotSameFileType && - firstFileNameWithoutSuffix === secondFileNameWithoutSuffix - ) { - // Also check that the size of an individual Live Photo asset is less - // than an (arbitrary) limit. This should be true in practice as the - // videos for a live photo are a few seconds long. Further on, the - // zipping library that we use doesn't support stream as a input. - const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ + if (fPrunedName != gPrunedName) return false; - if (f.size <= maxAssetSize && g.size <= maxAssetSize) { - return true; - } else { - log.info( - `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, - ); - } + // Also check that the size of an individual Live Photo asset is less than + // an (arbitrary) limit. This should be true in practice as the videos for a + // live photo are a few seconds long. Further on, the zipping library that + // we use doesn't support stream as a input. + + const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ + if (f.size > maxAssetSize || g.size > maxAssetSize) { + log.info( + `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, + ); + return false; } - return false; + + return true; }; const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { @@ -460,21 +460,6 @@ const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { return foundSuffix ? name.slice(0, foundSuffix.length * -1) : name; }; -function getFileNameWithoutExtension(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return filename; - else return filename.slice(0, lastDotPosition); -} - -function getFileExtensionWithDot(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return ""; - else return filename.slice(lastDotPosition); -} - -const isImageOrVideo = (fileType: FILE_TYPE) => - [FILE_TYPE.IMAGE, FILE_TYPE.VIDEO].includes(fileType); - async function getFileHash( worker: Remote, file: File | ElectronFile, From 3b0433c4abdbbc8a995262b58e55370c93beac0c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:14:53 +0530 Subject: [PATCH 63/82] Reorder --- .../src/services/upload/metadataService.ts | 174 +----------------- .../src/services/upload/uploadManager.ts | 165 ++++++++++++++++- 2 files changed, 169 insertions(+), 170 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index 22dc7d61f4..1e386b2d56 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -1,4 +1,4 @@ -import { getFileNameSize, nameAndExtension } from "@/next/file"; +import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; @@ -21,12 +21,9 @@ import { Metadata, ParsedExtractedMetadata, type DataStream, - type FileWithCollection, - type FileWithCollection2, type LivePhotoAssets2, type UploadAsset2, } from "types/upload"; -import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getEXIFLocation, getEXIFTime, getParsedExifData } from "./exifService"; import { MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, @@ -295,171 +292,6 @@ async function extractLivePhotoMetadata( }; } -export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { - return livePhotoAssets.image.size + livePhotoAssets.video.size; -} - -/** - * Go through the given files, combining any sibling image + video assets into a - * single live photo when appropriate. - */ -export const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { - const result: FileWithCollection2[] = []; - mediaFiles - .sort((f, g) => - nameAndExtension(getFileName(f.file))[0].localeCompare( - nameAndExtension(getFileName(g.file))[0], - ), - ) - .sort((f, g) => f.collectionID - g.collectionID); - let index = 0; - while (index < mediaFiles.length - 1) { - const f = mediaFiles[index]; - const g = mediaFiles[index + 1]; - const fFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(f.file), - ); - const gFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(g.file), - ); - const fa: PotentialLivePhotoAsset = { - collectionID: f.collectionID, - fileType: fFileType, - fileName: getFileName(f.file), - /* TODO(MR): ElectronFile changes */ - size: (f as FileWithCollection).file.size, - }; - const ga: PotentialLivePhotoAsset = { - collectionID: g.collectionID, - fileType: gFileType, - fileName: getFileName(g.file), - /* TODO(MR): ElectronFile changes */ - size: (g as FileWithCollection).file.size, - }; - if (areLivePhotoAssets(fa, ga)) { - let imageFile: File | ElectronFile | string; - let videoFile: File | ElectronFile | string; - if ( - fFileType === FILE_TYPE.IMAGE && - gFileType === FILE_TYPE.VIDEO - ) { - imageFile = f.file; - videoFile = g.file; - } else { - videoFile = f.file; - imageFile = g.file; - } - const livePhotoLocalID = f.localID; - result.push({ - localID: livePhotoLocalID, - collectionID: f.collectionID, - isLivePhoto: true, - livePhotoAssets: { - image: imageFile, - video: videoFile, - }, - }); - index += 2; - } else { - result.push({ - ...f, - isLivePhoto: false, - }); - index += 1; - } - } - if (index === mediaFiles.length - 1) { - result.push({ - ...mediaFiles[index], - isLivePhoto: false, - }); - } - return result; -}; - -interface PotentialLivePhotoAsset { - collectionID: number; - fileType: FILE_TYPE; - fileName: string; - size: number; -} - -const areLivePhotoAssets = ( - f: PotentialLivePhotoAsset, - g: PotentialLivePhotoAsset, -) => { - if (f.collectionID != g.collectionID) return false; - - const [fName, fExt] = nameAndExtension(f.fileName); - const [gName, gExt] = nameAndExtension(g.fileName); - - let fPrunedName: string; - let gPrunedName: string; - if (f.fileType == FILE_TYPE.IMAGE && g.fileType == FILE_TYPE.VIDEO) { - fPrunedName = removePotentialLivePhotoSuffix( - fName, - // A Google Live Photo image file can have video extension appended - // as suffix, so we pass that to removePotentialLivePhotoSuffix to - // remove it. - // - // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) - gExt ? `.${gExt}` : undefined, - ); - gPrunedName = removePotentialLivePhotoSuffix(gName); - } else if (f.fileType == FILE_TYPE.VIDEO && g.fileType == FILE_TYPE.IMAGE) { - fPrunedName = removePotentialLivePhotoSuffix(fName); - gPrunedName = removePotentialLivePhotoSuffix( - gName, - fExt ? `.${fExt}` : undefined, - ); - } else { - return false; - } - - if (fPrunedName != gPrunedName) return false; - - // Also check that the size of an individual Live Photo asset is less than - // an (arbitrary) limit. This should be true in practice as the videos for a - // live photo are a few seconds long. Further on, the zipping library that - // we use doesn't support stream as a input. - - const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ - if (f.size > maxAssetSize || g.size > maxAssetSize) { - log.info( - `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, - ); - return false; - } - - return true; -}; - -const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { - const suffix_3 = "_3"; - - // The icloud-photos-downloader library appends _HVEC to the end of the - // filename in case of live photos. - // - // https://github.com/icloud-photos-downloader/icloud_photos_downloader - const suffix_hvec = "_HVEC"; - - let foundSuffix: string | undefined; - if (name.endsWith(suffix_3)) { - foundSuffix = suffix_3; - } else if ( - name.endsWith(suffix_hvec) || - name.endsWith(suffix_hvec.toLowerCase()) - ) { - foundSuffix = suffix_hvec; - } else if (suffix) { - if (name.endsWith(suffix) || name.endsWith(suffix.toLowerCase())) { - foundSuffix = suffix; - } - } - - return foundSuffix ? name.slice(0, foundSuffix.length * -1) : name; -}; - async function getFileHash( worker: Remote, file: File | ElectronFile, @@ -499,3 +331,7 @@ async function getFileHash( log.info(`file hashing failed ${getFileNameSize(file)} ,${e.message} `); } } + +export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { + return livePhotoAssets.image.size + livePhotoAssets.video.size; +} diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 04080c62c9..fd9307ae8d 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,4 +1,5 @@ import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; @@ -9,6 +10,7 @@ import { Events, eventBus } from "@ente/shared/events"; import { wait } from "@ente/shared/utils"; import { Canceler } from "axios"; import { Remote } from "comlink"; +import { FILE_TYPE } from "constants/file"; import { RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UPLOAD_RESULT, @@ -43,7 +45,6 @@ import { segregateMetadataAndMediaFiles2, } from "utils/upload"; import { getLocalFiles } from "../fileService"; -import { clusterLivePhotoFiles, clusterLivePhotos } from "./metadataService"; import { getMetadataJSONMapKeyForJSON, tryParseTakeoutMetadataJSON, @@ -56,6 +57,7 @@ import UploadService, { getFileName, uploader, } from "./uploadService"; +import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; const MAX_CONCURRENT_UPLOADS = 4; @@ -800,3 +802,164 @@ const cancelRemainingUploads = async () => { await electron.setPendingUploadFiles("zips", []); await electron.setPendingUploadFiles("files", []); }; + +/** + * Go through the given files, combining any sibling image + video assets into a + * single live photo when appropriate. + */ +const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { + const result: FileWithCollection2[] = []; + mediaFiles + .sort((f, g) => + nameAndExtension(getFileName(f.file))[0].localeCompare( + nameAndExtension(getFileName(g.file))[0], + ), + ) + .sort((f, g) => f.collectionID - g.collectionID); + let index = 0; + while (index < mediaFiles.length - 1) { + const f = mediaFiles[index]; + const g = mediaFiles[index + 1]; + const fFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(f.file), + ); + const gFileType = getFileTypeFromExtensionForLivePhotoClustering( + getFileName(g.file), + ); + const fa: PotentialLivePhotoAsset = { + collectionID: f.collectionID, + fileType: fFileType, + fileName: getFileName(f.file), + /* TODO(MR): ElectronFile changes */ + size: (f as FileWithCollection).file.size, + }; + const ga: PotentialLivePhotoAsset = { + collectionID: g.collectionID, + fileType: gFileType, + fileName: getFileName(g.file), + /* TODO(MR): ElectronFile changes */ + size: (g as FileWithCollection).file.size, + }; + if (areLivePhotoAssets(fa, ga)) { + let imageFile: File | ElectronFile | string; + let videoFile: File | ElectronFile | string; + if ( + fFileType === FILE_TYPE.IMAGE && + gFileType === FILE_TYPE.VIDEO + ) { + imageFile = f.file; + videoFile = g.file; + } else { + videoFile = f.file; + imageFile = g.file; + } + const livePhotoLocalID = f.localID; + result.push({ + localID: livePhotoLocalID, + collectionID: f.collectionID, + isLivePhoto: true, + livePhotoAssets: { + image: imageFile, + video: videoFile, + }, + }); + index += 2; + } else { + result.push({ + ...f, + isLivePhoto: false, + }); + index += 1; + } + } + if (index === mediaFiles.length - 1) { + result.push({ + ...mediaFiles[index], + isLivePhoto: false, + }); + } + return result; +}; + +interface PotentialLivePhotoAsset { + collectionID: number; + fileType: FILE_TYPE; + fileName: string; + size: number; +} + +const areLivePhotoAssets = ( + f: PotentialLivePhotoAsset, + g: PotentialLivePhotoAsset, +) => { + if (f.collectionID != g.collectionID) return false; + + const [fName, fExt] = nameAndExtension(f.fileName); + const [gName, gExt] = nameAndExtension(g.fileName); + + let fPrunedName: string; + let gPrunedName: string; + if (f.fileType == FILE_TYPE.IMAGE && g.fileType == FILE_TYPE.VIDEO) { + fPrunedName = removePotentialLivePhotoSuffix( + fName, + // A Google Live Photo image file can have video extension appended + // as suffix, so we pass that to removePotentialLivePhotoSuffix to + // remove it. + // + // Example: IMG_20210630_0001.mp4.jpg (Google Live Photo image file) + gExt ? `.${gExt}` : undefined, + ); + gPrunedName = removePotentialLivePhotoSuffix(gName); + } else if (f.fileType == FILE_TYPE.VIDEO && g.fileType == FILE_TYPE.IMAGE) { + fPrunedName = removePotentialLivePhotoSuffix(fName); + gPrunedName = removePotentialLivePhotoSuffix( + gName, + fExt ? `.${fExt}` : undefined, + ); + } else { + return false; + } + + if (fPrunedName != gPrunedName) return false; + + // Also check that the size of an individual Live Photo asset is less than + // an (arbitrary) limit. This should be true in practice as the videos for a + // live photo are a few seconds long. Further on, the zipping library that + // we use doesn't support stream as a input. + + const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ + if (f.size > maxAssetSize || g.size > maxAssetSize) { + log.info( + `Not classifying assets with too large sizes ${[f.size, g.size]} as a live photo`, + ); + return false; + } + + return true; +}; + +const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { + const suffix_3 = "_3"; + + // The icloud-photos-downloader library appends _HVEC to the end of the + // filename in case of live photos. + // + // https://github.com/icloud-photos-downloader/icloud_photos_downloader + const suffix_hvec = "_HVEC"; + + let foundSuffix: string | undefined; + if (name.endsWith(suffix_3)) { + foundSuffix = suffix_3; + } else if ( + name.endsWith(suffix_hvec) || + name.endsWith(suffix_hvec.toLowerCase()) + ) { + foundSuffix = suffix_hvec; + } else if (suffix) { + if (name.endsWith(suffix) || name.endsWith(suffix.toLowerCase())) { + foundSuffix = suffix; + } + } + + return foundSuffix ? name.slice(0, foundSuffix.length * -1) : name; +}; From 58d2670171a9d16deb21d4f5acfac11fc3b5de89 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:29:54 +0530 Subject: [PATCH 64/82] Prune --- .../src/services/upload/uploadManager.ts | 43 ++++++----------- web/apps/photos/src/utils/file/livePhoto.ts | 42 ----------------- web/packages/media/file.ts | 6 +++ web/packages/media/live-photo.ts | 47 +++++++++++++++++++ 4 files changed, 67 insertions(+), 71 deletions(-) delete mode 100644 web/apps/photos/src/utils/file/livePhoto.ts create mode 100644 web/packages/media/file.ts diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index fd9307ae8d..27888467d2 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,3 +1,4 @@ +import { potentialFileTypeFromExtension } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; @@ -57,7 +58,6 @@ import UploadService, { getFileName, uploader, } from "./uploadService"; -import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; const MAX_CONCURRENT_UPLOADS = 4; @@ -820,47 +820,32 @@ const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { while (index < mediaFiles.length - 1) { const f = mediaFiles[index]; const g = mediaFiles[index + 1]; - const fFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(f.file), - ); - const gFileType = getFileTypeFromExtensionForLivePhotoClustering( - getFileName(g.file), - ); + const fFileName = getFileName(f.file); + const gFileName = getFileName(g.file); + const fFileType = potentialFileTypeFromExtension(fFileName); + const gFileType = potentialFileTypeFromExtension(gFileName); const fa: PotentialLivePhotoAsset = { - collectionID: f.collectionID, + fileName: fFileName, fileType: fFileType, - fileName: getFileName(f.file), + collectionID: f.collectionID, /* TODO(MR): ElectronFile changes */ size: (f as FileWithCollection).file.size, }; const ga: PotentialLivePhotoAsset = { - collectionID: g.collectionID, + fileName: gFileName, fileType: gFileType, - fileName: getFileName(g.file), + collectionID: g.collectionID, /* TODO(MR): ElectronFile changes */ size: (g as FileWithCollection).file.size, }; if (areLivePhotoAssets(fa, ga)) { - let imageFile: File | ElectronFile | string; - let videoFile: File | ElectronFile | string; - if ( - fFileType === FILE_TYPE.IMAGE && - gFileType === FILE_TYPE.VIDEO - ) { - imageFile = f.file; - videoFile = g.file; - } else { - videoFile = f.file; - imageFile = g.file; - } - const livePhotoLocalID = f.localID; result.push({ - localID: livePhotoLocalID, + localID: f.localID, collectionID: f.collectionID, isLivePhoto: true, livePhotoAssets: { - image: imageFile, - video: videoFile, + image: fFileType == FILE_TYPE.IMAGE ? f.file : g.file, + video: fFileType == FILE_TYPE.IMAGE ? g.file : f.file, }, }); index += 2; @@ -882,9 +867,9 @@ const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { }; interface PotentialLivePhotoAsset { - collectionID: number; - fileType: FILE_TYPE; fileName: string; + fileType: FILE_TYPE; + collectionID: number; size: number; } diff --git a/web/apps/photos/src/utils/file/livePhoto.ts b/web/apps/photos/src/utils/file/livePhoto.ts deleted file mode 100644 index 7d687217ce..0000000000 --- a/web/apps/photos/src/utils/file/livePhoto.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { FILE_TYPE } from "constants/file"; -import { getFileExtension } from "utils/file"; - -const IMAGE_EXTENSIONS = [ - "heic", - "heif", - "jpeg", - "jpg", - "png", - "gif", - "bmp", - "tiff", - "webp", -]; - -const VIDEO_EXTENSIONS = [ - "mov", - "mp4", - "m4v", - "avi", - "wmv", - "flv", - "mkv", - "webm", - "3gp", - "3g2", - "avi", - "ogv", - "mpg", - "mp", -]; - -export function getFileTypeFromExtensionForLivePhotoClustering( - filename: string, -) { - const extension = getFileExtension(filename)?.toLowerCase(); - if (IMAGE_EXTENSIONS.includes(extension)) { - return FILE_TYPE.IMAGE; - } else if (VIDEO_EXTENSIONS.includes(extension)) { - return FILE_TYPE.VIDEO; - } -} diff --git a/web/packages/media/file.ts b/web/packages/media/file.ts new file mode 100644 index 0000000000..8917db8025 --- /dev/null +++ b/web/packages/media/file.ts @@ -0,0 +1,6 @@ +export enum FILE_TYPE { + IMAGE, + VIDEO, + LIVE_PHOTO, + OTHERS, +} diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index 16143ca138..c4a6314673 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -1,5 +1,52 @@ import { fileNameFromComponents, nameAndExtension } from "@/next/file"; import JSZip from "jszip"; +import { FILE_TYPE } from "./file"; + +const potentialImageExtensions = [ + "heic", + "heif", + "jpeg", + "jpg", + "png", + "gif", + "bmp", + "tiff", + "webp", +]; + +const potentialVideoExtensions = [ + "mov", + "mp4", + "m4v", + "avi", + "wmv", + "flv", + "mkv", + "webm", + "3gp", + "3g2", + "avi", + "ogv", + "mpg", + "mp", +]; + +/** + * Use the file extension of the given {@link fileName} to deduce if is is + * potentially the image or the video part of a Live Photo. + */ +export const potentialFileTypeFromExtension = ( + fileName: string, +): FILE_TYPE | undefined => { + let [, ext] = nameAndExtension(fileName); + if (!ext) return undefined; + + ext = ext.toLowerCase(); + + if (potentialImageExtensions.includes(ext)) return FILE_TYPE.IMAGE; + else if (potentialVideoExtensions.includes(ext)) return FILE_TYPE.VIDEO; + else return undefined; +}; /** * An in-memory representation of a live photo. From e490f194e76e982117afc10f4b1579b3d718019a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:31:16 +0530 Subject: [PATCH 65/82] Line --- web/apps/photos/src/services/upload/uploadService.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index f1583dabff..3d496718dd 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -56,6 +56,7 @@ import { getLivePhotoSize, } from "./metadataService"; import publicUploadHttpClient from "./publicUploadHttpClient"; +import type { ParsedMetadataJSON } from "./takeout"; import { fallbackThumbnail, generateThumbnailNative, @@ -63,7 +64,6 @@ import { } from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; -import type { ParsedMetadataJSON } from "./takeout"; /** Upload files to cloud storage */ class UploadService { @@ -264,6 +264,7 @@ export const uploader = async ( encryptedFile.file, makeProgessTracker, isCFUploadProxyDisabled, + abortIfCancelled, ); const uploadedFile = await uploadService.uploadFile({ @@ -766,6 +767,7 @@ const uploadToBucket = async ( file: ProcessedFile, makeProgessTracker: MakeProgressTracker, isCFUploadProxyDisabled: boolean, + abortIfCancelled: () => void, ): Promise => { try { let fileObjectKey: string = null; @@ -776,6 +778,7 @@ const uploadToBucket = async ( file.file.encryptedData, makeProgessTracker, isCFUploadProxyDisabled, + abortIfCancelled, ); } else { const progressTracker = makeProgessTracker(file.localID); @@ -841,6 +844,7 @@ async function uploadStreamUsingMultipart( dataStream: DataStream, makeProgessTracker: MakeProgressTracker, isCFUploadProxyDisabled: boolean, + abortIfCancelled: () => void, ) { const uploadPartCount = Math.ceil( dataStream.chunkCount / FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, @@ -858,9 +862,8 @@ async function uploadStreamUsingMultipart( index, fileUploadURL, ] of multipartUploadURLs.partURLs.entries()) { - if (uploadCancelService.isUploadCancelationRequested()) { - throw Error(CustomError.UPLOAD_CANCELLED); - } + abortIfCancelled(); + const uploadChunk = await combineChunksToFormUploadPart(streamReader); const progressTracker = makeProgessTracker( fileLocalID, From feb59b00d2b9e1ee979782a366c1c3871918dabb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:32:31 +0530 Subject: [PATCH 66/82] Move --- .../src/services/upload/metadataService.ts | 23 ++---------------- .../src/services/upload/uploadService.ts | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index 1e386b2d56..ad391eb456 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -13,11 +13,9 @@ import { FILE_TYPE } from "constants/file"; import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload"; import * as ffmpegService from "services/ffmpeg"; import { getElectronFileStream, getFileStream } from "services/readerService"; -import { getFileType } from "services/typeDetectionService"; import { FilePublicMagicMetadataProps } from "types/file"; import { FileTypeInfo, - LivePhotoAssets, Metadata, ParsedExtractedMetadata, type DataStream, @@ -95,7 +93,7 @@ export async function extractMetadata( return { metadata, publicMagicMetadata }; } -export async function getImageMetadata( +async function getImageMetadata( receivedFile: File | ElectronFile, fileTypeInfo: FileTypeInfo, ): Promise { @@ -129,7 +127,7 @@ export async function getImageMetadata( } // tries to extract date from file name if available else returns null -export function extractDateFromFileName(filename: string): number { +function extractDateFromFileName(filename: string): number { try { filename = filename.trim(); let parsedDate: Date; @@ -184,19 +182,6 @@ async function getVideoMetadata(file: File | ElectronFile) { return videoMetadata; } -export async function getLivePhotoFileType( - livePhotoAssets: LivePhotoAssets, -): Promise { - const imageFileTypeInfo = await getFileType(livePhotoAssets.image); - const videoFileTypeInfo = await getFileType(livePhotoAssets.video); - return { - fileType: FILE_TYPE.LIVE_PHOTO, - exactType: `${imageFileTypeInfo.exactType}+${videoFileTypeInfo.exactType}`, - imageType: imageFileTypeInfo.exactType, - videoType: videoFileTypeInfo.exactType, - }; -} - export const extractAssetMetadata = async ( worker: Remote, parsedMetadataJSONMap: Map, @@ -331,7 +316,3 @@ async function getFileHash( log.info(`file hashing failed ${getFileNameSize(file)} ,${e.message} `); } } - -export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { - return livePhotoAssets.image.size + livePhotoAssets.video.size; -} diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 3d496718dd..baa57c7e6c 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -37,6 +37,7 @@ import { isDataStream, type DataStream, type FileWithCollection2, + type LivePhotoAssets, type LivePhotoAssets2, type Metadata, type UploadAsset2, @@ -50,11 +51,7 @@ import { hasFileHash } from "utils/upload"; import * as convert from "xml-js"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; -import { - extractAssetMetadata, - getLivePhotoFileType, - getLivePhotoSize, -} from "./metadataService"; +import { extractAssetMetadata, getLivePhotoFileType } from "./metadataService"; import publicUploadHttpClient from "./publicUploadHttpClient"; import type { ParsedMetadataJSON } from "./takeout"; import { @@ -330,6 +327,10 @@ const getAssetSize = ({ isLivePhoto, file, livePhotoAssets }: UploadAsset) => { return isLivePhoto ? getLivePhotoSize(livePhotoAssets) : getFileSize(file); }; +const getLivePhotoSize = (livePhotoAssets: LivePhotoAssets) => { + return livePhotoAssets.image.size + livePhotoAssets.video.size; +}; + const getAssetFileType = ({ isLivePhoto, file, @@ -340,6 +341,19 @@ const getAssetFileType = ({ : getFileType(file); }; +const getLivePhotoFileType = async ( + livePhotoAssets: LivePhotoAssets, +): Promise => { + const imageFileTypeInfo = await getFileType(livePhotoAssets.image); + const videoFileTypeInfo = await getFileType(livePhotoAssets.video); + return { + fileType: FILE_TYPE.LIVE_PHOTO, + exactType: `${imageFileTypeInfo.exactType}+${videoFileTypeInfo.exactType}`, + imageType: imageFileTypeInfo.exactType, + videoType: videoFileTypeInfo.exactType, + }; +}; + const readAsset = async ( fileTypeInfo: FileTypeInfo, { isLivePhoto, file, livePhotoAssets }: UploadAsset2, From 00c0780de17a8b235878b4df964a3f39af77418f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:40:35 +0530 Subject: [PATCH 67/82] Reorder --- .../src/services/upload/metadataService.ts | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadataService.ts index ad391eb456..e4f4c348b6 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadataService.ts @@ -61,7 +61,63 @@ export interface ExtractMetadataResult { publicMagicMetadata: FilePublicMagicMetadataProps; } -export async function extractMetadata( +export const extractAssetMetadata = async ( + worker: Remote, + parsedMetadataJSONMap: Map, + { isLivePhoto, file, livePhotoAssets }: UploadAsset2, + collectionID: number, + fileTypeInfo: FileTypeInfo, +): Promise => { + return isLivePhoto + ? await extractLivePhotoMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + livePhotoAssets, + ) + : await extractFileMetadata( + worker, + parsedMetadataJSONMap, + collectionID, + fileTypeInfo, + file, + ); +}; + +async function extractFileMetadata( + worker: Remote, + parsedMetadataJSONMap: Map, + collectionID: number, + fileTypeInfo: FileTypeInfo, + rawFile: File | ElectronFile | string, +): Promise { + const rawFileName = getFileName(rawFile); + let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); + let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); + + if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { + key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); + googleMetadata = parsedMetadataJSONMap.get(key); + } + + const { metadata, publicMagicMetadata } = await extractMetadata( + worker, + /* TODO(MR): ElectronFile changes */ + rawFile as File | ElectronFile, + fileTypeInfo, + ); + + for (const [key, value] of Object.entries(googleMetadata ?? {})) { + if (!value) { + continue; + } + metadata[key] = value; + } + return { metadata, publicMagicMetadata }; +} + +async function extractMetadata( worker: Remote, receivedFile: File | ElectronFile, fileTypeInfo: FileTypeInfo, @@ -182,62 +238,6 @@ async function getVideoMetadata(file: File | ElectronFile) { return videoMetadata; } -export const extractAssetMetadata = async ( - worker: Remote, - parsedMetadataJSONMap: Map, - { isLivePhoto, file, livePhotoAssets }: UploadAsset2, - collectionID: number, - fileTypeInfo: FileTypeInfo, -): Promise => { - return isLivePhoto - ? await extractLivePhotoMetadata( - worker, - parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - livePhotoAssets, - ) - : await extractFileMetadata( - worker, - parsedMetadataJSONMap, - collectionID, - fileTypeInfo, - file, - ); -}; - -async function extractFileMetadata( - worker: Remote, - parsedMetadataJSONMap: Map, - collectionID: number, - fileTypeInfo: FileTypeInfo, - rawFile: File | ElectronFile | string, -): Promise { - const rawFileName = getFileName(rawFile); - let key = getMetadataJSONMapKeyForFile(collectionID, rawFileName); - let googleMetadata: ParsedMetadataJSON = parsedMetadataJSONMap.get(key); - - if (!googleMetadata && key.length > MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT) { - key = getClippedMetadataJSONMapKeyForFile(collectionID, rawFileName); - googleMetadata = parsedMetadataJSONMap.get(key); - } - - const { metadata, publicMagicMetadata } = await extractMetadata( - worker, - /* TODO(MR): ElectronFile changes */ - rawFile as File | ElectronFile, - fileTypeInfo, - ); - - for (const [key, value] of Object.entries(googleMetadata ?? {})) { - if (!value) { - continue; - } - metadata[key] = value; - } - return { metadata, publicMagicMetadata }; -} - async function extractLivePhotoMetadata( worker: Remote, parsedMetadataJSONMap: Map, From cb0b54902752df28cbaff686b70de1a6c6a61088 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:42:35 +0530 Subject: [PATCH 68/82] Rename --- .../src/services/upload/{metadataService.ts => metadata.ts} | 4 ++-- web/apps/photos/src/services/upload/uploadService.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename web/apps/photos/src/services/upload/{metadataService.ts => metadata.ts} (98%) diff --git a/web/apps/photos/src/services/upload/metadataService.ts b/web/apps/photos/src/services/upload/metadata.ts similarity index 98% rename from web/apps/photos/src/services/upload/metadataService.ts rename to web/apps/photos/src/services/upload/metadata.ts index e4f4c348b6..48283b34b5 100644 --- a/web/apps/photos/src/services/upload/metadataService.ts +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -49,14 +49,14 @@ const EXIF_TAGS_NEEDED = [ "MetadataDate", ]; -export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { +const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { location: NULL_LOCATION, creationTime: null, width: null, height: null, }; -export interface ExtractMetadataResult { +interface ExtractMetadataResult { metadata: Metadata; publicMagicMetadata: FilePublicMagicMetadataProps; } diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index baa57c7e6c..5e4ee0b008 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -51,7 +51,7 @@ import { hasFileHash } from "utils/upload"; import * as convert from "xml-js"; import { getFileStream } from "../readerService"; import { getFileType } from "../typeDetectionService"; -import { extractAssetMetadata, getLivePhotoFileType } from "./metadataService"; +import { extractAssetMetadata } from "./metadata"; import publicUploadHttpClient from "./publicUploadHttpClient"; import type { ParsedMetadataJSON } from "./takeout"; import { From 58b1c4b489f6b876408adcf42399b32b2c3da2b9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 11:49:58 +0530 Subject: [PATCH 69/82] Doodle --- web/apps/photos/src/services/upload/metadata.ts | 4 ++-- web/apps/photos/src/types/upload/index.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/upload/metadata.ts b/web/apps/photos/src/services/upload/metadata.ts index 48283b34b5..c3ccea0e28 100644 --- a/web/apps/photos/src/services/upload/metadata.ts +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -128,7 +128,7 @@ async function extractMetadata( } else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) { extractedMetadata = await getVideoMetadata(receivedFile); } - const fileHash = await getFileHash(worker, receivedFile); + const hash = await getFileHash(worker, receivedFile); const metadata: Metadata = { title: receivedFile.name, @@ -140,7 +140,7 @@ async function extractMetadata( latitude: extractedMetadata.location.latitude, longitude: extractedMetadata.location.longitude, fileType: fileTypeInfo.fileType, - hash: fileHash, + hash, }; const publicMagicMetadata: FilePublicMagicMetadataProps = { w: extractedMetadata.width, diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 95913531f3..aef7474a4c 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -12,6 +12,7 @@ import { } from "types/file"; import { EncryptedMagicMetadata } from "types/magicMetadata"; +/** Information about the file that never changes post upload. */ export interface Metadata { /** * The file name. From 06dbf5fb0678d3a3033bc1680082688edf205a87 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 12:21:52 +0530 Subject: [PATCH 70/82] Phasal types --- .../src/services/upload/uploadManager.ts | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 27888467d2..1a65b1be9f 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -31,6 +31,8 @@ import { FileWithCollection, PublicUploadProps, type FileWithCollection2, + type LivePhotoAssets, + type LivePhotoAssets2, } from "types/upload"; import { FinishedUploads, @@ -803,12 +805,30 @@ const cancelRemainingUploads = async () => { await electron.setPendingUploadFiles("files", []); }; +/** + * The data needed by {@link clusterLivePhotos} to do its thing. + * + * As files progress through stages, they get more and more bits tacked on to + * them. These types document the journey. + */ +type ClusterableFile = { + localID: number; + collectionID: number; + // fileOrPath: File | ElectronFile | string; + file: File | ElectronFile | string; +} + +type ClusteredFile = ClusterableFile & { + isLivePhoto: boolean; + livePhotoAssets?: LivePhotoAssets2; +} + /** * Go through the given files, combining any sibling image + video assets into a * single live photo when appropriate. */ -const clusterLivePhotos = (mediaFiles: FileWithCollection2[]) => { - const result: FileWithCollection2[] = []; +const clusterLivePhotos = (mediaFiles: ClusterableFile[]) => { + const result: ClusteredFile[] = []; mediaFiles .sort((f, g) => nameAndExtension(getFileName(f.file))[0].localeCompare( From 239688b7d82e972e76304f7fc5fa301b4a329689 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 12:27:33 +0530 Subject: [PATCH 71/82] Remove potentially dangerous wip forks --- .../photos/src/components/Upload/Uploader.tsx | 77 +-------------- .../src/services/upload/uploadManager.ts | 93 +------------------ web/apps/photos/src/utils/upload/index.ts | 17 ---- 3 files changed, 3 insertions(+), 184 deletions(-) diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index 7b4bc4fa34..c340ee1dc3 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -489,7 +489,7 @@ export default function Uploader(props: Props) { }); throw e; } - await waitInQueueAndUploadFiles2( + await waitInQueueAndUploadFiles( filesWithCollectionToUpload, collections, ); @@ -517,24 +517,6 @@ export default function Uploader(props: Props) { await currentUploadPromise.current; }; - const waitInQueueAndUploadFiles2 = async ( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) => { - const currentPromise = currentUploadPromise.current; - currentUploadPromise.current = waitAndRun( - currentPromise, - async () => - await uploadFiles2( - filesWithCollectionToUploadIn, - collections, - uploaderName, - ), - ); - await currentUploadPromise.current; - }; - const preUploadAction = async () => { uploadManager.prepareForNewUpload(); setUploadProgressView(true); @@ -604,63 +586,6 @@ export default function Uploader(props: Props) { } }; - const uploadFiles2 = async ( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) => { - try { - log.info("uploadFiles called"); - preUploadAction(); - if ( - electron && - !isPendingDesktopUpload.current && - !watcher.isUploadRunning() - ) { - await setToUploadCollection(collections); - if (zipPaths.current) { - await electron.setPendingUploadFiles( - "zips", - zipPaths.current, - ); - zipPaths.current = null; - } - await electron.setPendingUploadFiles( - "files", - filesWithCollectionToUploadIn.map( - ({ file }) => (file as ElectronFile).path, - ), - ); - } - const shouldCloseUploadProgress = - await uploadManager.queueFilesForUpload2( - filesWithCollectionToUploadIn, - collections, - uploaderName, - ); - if (shouldCloseUploadProgress) { - closeUploadProgress(); - } - if (isElectron()) { - if (watcher.isUploadRunning()) { - await watcher.allFileUploadsDone( - filesWithCollectionToUploadIn, - collections, - ); - } else if (watcher.isSyncPaused()) { - // resume the service after user upload is done - watcher.resumePausedSync(); - } - } - } catch (e) { - log.error("failed to upload files", e); - showUserFacingError(e.message); - closeUploadProgress(); - } finally { - postUploadAction(); - } - }; - const retryFailed = async () => { try { log.info("user retrying failed upload"); diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 1a65b1be9f..4a18c48c1e 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -31,7 +31,6 @@ import { FileWithCollection, PublicUploadProps, type FileWithCollection2, - type LivePhotoAssets, type LivePhotoAssets2, } from "types/upload"; import { @@ -45,7 +44,6 @@ import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file"; import { areFileWithCollectionsSame, segregateMetadataAndMediaFiles, - segregateMetadataAndMediaFiles2, } from "utils/upload"; import { getLocalFiles } from "../fileService"; import { @@ -423,93 +421,6 @@ class UploadManager { } } - public async queueFilesForUpload2( - filesWithCollectionToUploadIn: FileWithCollection2[], - collections: Collection[], - uploaderName?: string, - ) { - try { - if (this.uploadInProgress) { - throw Error("can't run multiple uploads at once"); - } - this.uploadInProgress = true; - await this.updateExistingFilesAndCollections(collections); - this.uploaderName = uploaderName; - log.info( - `received ${filesWithCollectionToUploadIn.length} files to upload`, - ); - this.uiService.setFilenames( - new Map( - filesWithCollectionToUploadIn.map((mediaFile) => [ - mediaFile.localID, - assetName(mediaFile), - ]), - ), - ); - const { metadataJSONFiles, mediaFiles } = - segregateMetadataAndMediaFiles2(filesWithCollectionToUploadIn); - log.info(`has ${metadataJSONFiles.length} metadata json files`); - log.info(`has ${mediaFiles.length} media files`); - if (metadataJSONFiles.length) { - this.uiService.setUploadStage( - UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, - ); - await this.parseMetadataJSONFiles(metadataJSONFiles); - } - if (mediaFiles.length) { - log.info(`clusterLivePhotoFiles started`); - const analysedMediaFiles = - await clusterLivePhotoFiles(mediaFiles); - log.info(`clusterLivePhotoFiles ended`); - log.info( - `got live photos: ${ - mediaFiles.length !== analysedMediaFiles.length - }`, - ); - this.uiService.setFilenames( - new Map( - analysedMediaFiles.map((mediaFile) => [ - mediaFile.localID, - assetName(mediaFile), - ]), - ), - ); - - this.uiService.setHasLivePhoto( - mediaFiles.length !== analysedMediaFiles.length, - ); - - await this.uploadMediaFiles(analysedMediaFiles); - } - } catch (e) { - if (e.message === CustomError.UPLOAD_CANCELLED) { - if (isElectron()) { - this.remainingFiles = []; - await cancelRemainingUploads(); - } - } else { - log.error("uploading failed with error", e); - throw e; - } - } finally { - this.uiService.setUploadStage(UPLOAD_STAGES.FINISH); - for (let i = 0; i < MAX_CONCURRENT_UPLOADS; i++) { - this.cryptoWorkers[i]?.terminate(); - } - this.uploadInProgress = false; - } - try { - if (!this.uiService.hasFilesInResultList()) { - return true; - } else { - return false; - } - } catch (e) { - log.error(" failed to return shouldCloseProgressBar", e); - return false; - } - } - private async parseMetadataJSONFiles(metadataFiles: FileWithCollection2[]) { try { log.info(`parseMetadataJSONFiles function executed `); @@ -816,12 +727,12 @@ type ClusterableFile = { collectionID: number; // fileOrPath: File | ElectronFile | string; file: File | ElectronFile | string; -} +}; type ClusteredFile = ClusterableFile & { isLivePhoto: boolean; livePhotoAssets?: LivePhotoAssets2; -} +}; /** * Go through the given files, combining any sibling image + video assets into a diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 6de421ca9b..091026e7be 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -30,23 +30,6 @@ export function segregateMetadataAndMediaFiles( return { mediaFiles, metadataJSONFiles }; } -export function segregateMetadataAndMediaFiles2( - filesWithCollectionToUpload: FileWithCollection2[], -) { - const metadataJSONFiles: FileWithCollection2[] = []; - const mediaFiles: FileWithCollection2[] = []; - filesWithCollectionToUpload.forEach((fileWithCollection) => { - const file = fileWithCollection.file; - const s = typeof file == "string" ? file : file.name; - if (s.toLowerCase().endsWith(TYPE_JSON)) { - metadataJSONFiles.push(fileWithCollection); - } else { - mediaFiles.push(fileWithCollection); - } - }); - return { mediaFiles, metadataJSONFiles }; -} - export function areFileWithCollectionsSame( firstFile: FileWithCollection2, secondFile: FileWithCollection2, From d96f710d6d27595b9acdc25462fa50ac3edcb859 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 12:35:27 +0530 Subject: [PATCH 72/82] Prune --- .../photos/src/services/upload/uploadManager.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 4a18c48c1e..0c54d0f02d 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -41,10 +41,7 @@ import { SegregatedFinishedUploads, } from "types/upload/ui"; import { decryptFile, getUserOwnedFiles, sortFiles } from "utils/file"; -import { - areFileWithCollectionsSame, - segregateMetadataAndMediaFiles, -} from "utils/upload"; +import { segregateMetadataAndMediaFiles } from "utils/upload"; import { getLocalFiles } from "../fileService"; import { getMetadataJSONMapKeyForJSON, @@ -552,7 +549,7 @@ class UploadManager { log.info( `post upload action -> fileUploadResult: ${fileUploadResult} uploadedFile present ${!!uploadedFile}`, ); - await this.updateElectronRemainingFiles(fileWithCollection); + await this.removeFromPendingUploads(fileWithCollection); switch (fileUploadResult) { case UPLOAD_RESULT.FAILED: case UPLOAD_RESULT.BLOCKED: @@ -655,12 +652,10 @@ class UploadManager { this.setFiles((files) => sortFiles([...files, decryptedFile])); } - private async updateElectronRemainingFiles( - fileWithCollection: FileWithCollection2, - ) { + private async removeFromPendingUploads(file: FileWithCollection2) { if (isElectron()) { this.remainingFiles = this.remainingFiles.filter( - (file) => !areFileWithCollectionsSame(file, fileWithCollection), + (f) => f.localID != file.localID, ); await updatePendingUploads(this.remainingFiles); } From 51a1c77720c37361ba6d39438ed6563c768e6109 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 12:46:11 +0530 Subject: [PATCH 73/82] Get to a mergeable state --- web/apps/photos/src/services/ffmpeg.ts | 3 +++ web/apps/photos/src/services/upload/metadata.ts | 2 +- web/apps/photos/src/services/upload/uploadService.ts | 3 +-- web/apps/photos/src/types/upload/index.ts | 10 +--------- web/apps/photos/tests/zip-file-reading.test.ts | 2 +- web/packages/shared/crypto/types.ts | 2 +- web/packages/shared/utils/data-stream.ts | 8 ++++++++ 7 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 web/packages/shared/utils/data-stream.ts diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 2778efb26f..58b182e73b 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -212,6 +212,8 @@ const ffmpegExecWeb = async ( * * See also: {@link ffmpegExecWeb}. */ +/* +TODO(MR): Remove me const ffmpegExecNative = async ( electron: Electron, command: string[], @@ -227,6 +229,7 @@ const ffmpegExecNative = async ( return await worker.exec(command, blob, timeoutMs); } }; +*/ const ffmpegExec2 = async ( command: string[], diff --git a/web/apps/photos/src/services/upload/metadata.ts b/web/apps/photos/src/services/upload/metadata.ts index c3ccea0e28..4cda767a03 100644 --- a/web/apps/photos/src/services/upload/metadata.ts +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -18,7 +18,6 @@ import { FileTypeInfo, Metadata, ParsedExtractedMetadata, - type DataStream, type LivePhotoAssets2, type UploadAsset2, } from "types/upload"; @@ -30,6 +29,7 @@ import { type ParsedMetadataJSON, } from "./takeout"; import { getFileName } from "./uploadService"; +import type { DataStream } from "@ente/shared/utils/data-stream"; const EXIF_TAGS_NEEDED = [ "DateTimeOriginal", diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 5e4ee0b008..b27843186f 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -34,8 +34,6 @@ import { UploadAsset, UploadFile, UploadURL, - isDataStream, - type DataStream, type FileWithCollection2, type LivePhotoAssets, type LivePhotoAssets2, @@ -61,6 +59,7 @@ import { } from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; +import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream"; /** Upload files to cloud storage */ class UploadService { diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index aef7474a4c..c687911c71 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -3,6 +3,7 @@ import { B64EncryptionResult, LocalFileAttributes, } from "@ente/shared/crypto/types"; +import type { DataStream } from "@ente/shared/utils/data-stream"; import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { @@ -92,15 +93,6 @@ export interface UploadURL { objectKey: string; } -export interface DataStream { - stream: ReadableStream; - chunkCount: number; -} - -export function isDataStream(object: any): object is DataStream { - return "stream" in object; -} - export interface FileInMemory { filedata: Uint8Array | DataStream; /** The JPEG data of the generated thumbnail */ diff --git a/web/apps/photos/tests/zip-file-reading.test.ts b/web/apps/photos/tests/zip-file-reading.test.ts index 07d70f067d..ea7511d0b4 100644 --- a/web/apps/photos/tests/zip-file-reading.test.ts +++ b/web/apps/photos/tests/zip-file-reading.test.ts @@ -1,7 +1,7 @@ import { getFileNameSize } from "@/next/file"; +import type { DataStream } from "@ente/shared/utils/data-stream"; import { FILE_READER_CHUNK_SIZE, PICKED_UPLOAD_TYPE } from "constants/upload"; import { getElectronFileStream, getFileStream } from "services/readerService"; -import { DataStream } from "types/upload"; import { getImportSuggestion } from "utils/upload"; // This was for used to verify that converting from the browser readable stream diff --git a/web/packages/shared/crypto/types.ts b/web/packages/shared/crypto/types.ts index 4cf4c56b1f..47bfa8b2ce 100644 --- a/web/packages/shared/crypto/types.ts +++ b/web/packages/shared/crypto/types.ts @@ -1,4 +1,4 @@ -import { DataStream } from "@/next/types/file"; +import type { DataStream } from "../utils/data-stream"; export interface LocalFileAttributes< T extends string | Uint8Array | DataStream, diff --git a/web/packages/shared/utils/data-stream.ts b/web/packages/shared/utils/data-stream.ts new file mode 100644 index 0000000000..d072dfe7ec --- /dev/null +++ b/web/packages/shared/utils/data-stream.ts @@ -0,0 +1,8 @@ +export interface DataStream { + stream: ReadableStream; + chunkCount: number; +} + +export function isDataStream(object: any): object is DataStream { + return "stream" in object; +} From 0566d2ee93380373466cdd9b39e5d002b59de203 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:01:53 +0530 Subject: [PATCH 74/82] Spruce --- web/apps/cast/src/constants/file.ts | 20 ------------------- web/apps/cast/src/constants/upload.ts | 16 ++++++++++++++- web/apps/cast/src/pages/slideshow.tsx | 2 +- .../cast/src/services/castDownloadManager.ts | 2 +- web/apps/cast/src/services/readerService.ts | 14 ------------- .../cast/src/services/typeDetectionService.ts | 17 +++++++++++++--- web/apps/cast/src/types/upload.ts | 2 +- .../cast/src/utils/{file/index.ts => file.ts} | 15 ++------------ .../photos/src/services/upload/metadata.ts | 2 +- .../src/services/upload/uploadService.ts | 2 +- web/apps/photos/src/utils/file/index.ts | 12 +++++++++++ 11 files changed, 48 insertions(+), 56 deletions(-) delete mode 100644 web/apps/cast/src/constants/file.ts delete mode 100644 web/apps/cast/src/services/readerService.ts rename web/apps/cast/src/utils/{file/index.ts => file.ts} (88%) diff --git a/web/apps/cast/src/constants/file.ts b/web/apps/cast/src/constants/file.ts deleted file mode 100644 index 9be5746388..0000000000 --- a/web/apps/cast/src/constants/file.ts +++ /dev/null @@ -1,20 +0,0 @@ -export enum FILE_TYPE { - IMAGE, - VIDEO, - LIVE_PHOTO, - OTHERS, -} - -export const RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "raf", - "nef", - "psd", - "dng", - "tif", -]; diff --git a/web/apps/cast/src/constants/upload.ts b/web/apps/cast/src/constants/upload.ts index 63d044fb49..801d8a6ab4 100644 --- a/web/apps/cast/src/constants/upload.ts +++ b/web/apps/cast/src/constants/upload.ts @@ -1,6 +1,20 @@ -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { FileTypeInfo } from "types/upload"; +export const RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "raf", + "nef", + "psd", + "dng", + "tif", +]; + // list of format that were missed by type-detection for some files. export const WHITELISTED_FILE_FORMATS: FileTypeInfo[] = [ { fileType: FILE_TYPE.IMAGE, exactType: "jpeg", mimeType: "image/jpeg" }, diff --git a/web/apps/cast/src/pages/slideshow.tsx b/web/apps/cast/src/pages/slideshow.tsx index 774bbd4da9..000d18380b 100644 --- a/web/apps/cast/src/pages/slideshow.tsx +++ b/web/apps/cast/src/pages/slideshow.tsx @@ -1,7 +1,7 @@ import log from "@/next/log"; import PairedSuccessfullyOverlay from "components/PairedSuccessfullyOverlay"; import { PhotoAuditorium } from "components/PhotoAuditorium"; -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { diff --git a/web/apps/cast/src/services/castDownloadManager.ts b/web/apps/cast/src/services/castDownloadManager.ts index 76b37c082a..1b890fbbc1 100644 --- a/web/apps/cast/src/services/castDownloadManager.ts +++ b/web/apps/cast/src/services/castDownloadManager.ts @@ -2,7 +2,7 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getCastFileURL } from "@ente/shared/network/api"; -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { EnteFile } from "types/file"; import { generateStreamFromArrayBuffer } from "utils/file"; diff --git a/web/apps/cast/src/services/readerService.ts b/web/apps/cast/src/services/readerService.ts deleted file mode 100644 index 19f9bb9311..0000000000 --- a/web/apps/cast/src/services/readerService.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { convertBytesToHumanReadable } from "@/next/file"; -import log from "@/next/log"; - -export async function getUint8ArrayView(file: Blob): Promise { - try { - return new Uint8Array(await file.arrayBuffer()); - } catch (e) { - log.error( - `Failed to read file blob of size ${convertBytesToHumanReadable(file.size)}`, - e, - ); - throw e; - } -} diff --git a/web/apps/cast/src/services/typeDetectionService.ts b/web/apps/cast/src/services/typeDetectionService.ts index 5acd3844dc..883d96b0e1 100644 --- a/web/apps/cast/src/services/typeDetectionService.ts +++ b/web/apps/cast/src/services/typeDetectionService.ts @@ -1,14 +1,13 @@ -import { nameAndExtension } from "@/next/file"; +import { FILE_TYPE } from "@/media/file"; +import { convertBytesToHumanReadable, nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; -import { FILE_TYPE } from "constants/file"; import { KNOWN_NON_MEDIA_FORMATS, WHITELISTED_FILE_FORMATS, } from "constants/upload"; import FileType from "file-type"; import { FileTypeInfo } from "types/upload"; -import { getUint8ArrayView } from "./readerService"; const TYPE_VIDEO = "video"; const TYPE_IMAGE = "image"; @@ -66,6 +65,18 @@ async function extractFileType(file: File) { return getFileTypeFromBuffer(fileDataChunk); } +export async function getUint8ArrayView(file: Blob): Promise { + try { + return new Uint8Array(await file.arrayBuffer()); + } catch (e) { + log.error( + `Failed to read file blob of size ${convertBytesToHumanReadable(file.size)}`, + e, + ); + throw e; + } +} + async function getFileTypeFromBuffer(buffer: Uint8Array) { const result = await FileType.fromBuffer(buffer); if (!result?.mime) { diff --git a/web/apps/cast/src/types/upload.ts b/web/apps/cast/src/types/upload.ts index 6dc8818202..91b5c33851 100644 --- a/web/apps/cast/src/types/upload.ts +++ b/web/apps/cast/src/types/upload.ts @@ -1,4 +1,4 @@ -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; export interface Metadata { title: string; diff --git a/web/apps/cast/src/utils/file/index.ts b/web/apps/cast/src/utils/file.ts similarity index 88% rename from web/apps/cast/src/utils/file/index.ts rename to web/apps/cast/src/utils/file.ts index 60ec0e56e6..1f04a916df 100644 --- a/web/apps/cast/src/utils/file/index.ts +++ b/web/apps/cast/src/utils/file.ts @@ -1,7 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; -import { FILE_TYPE, RAW_FORMATS } from "constants/file"; +import { RAW_FORMATS } from "constants/upload"; import CastDownloadManager from "services/castDownloadManager"; import { getFileType } from "services/typeDetectionService"; import { @@ -103,18 +104,6 @@ export function isRawFileFromFileName(fileName: string) { return false; } -/** - * [Note: File name for local EnteFile objects] - * - * The title property in a file's metadata is the original file's name. The - * metadata of a file cannot be edited. So if later on the file's name is - * changed, then the edit is stored in the `editedName` property of the public - * metadata of the file. - * - * This function merges these edits onto the file object that we use locally. - * Effectively, post this step, the file's metadata.title can be used in lieu of - * its filename. - */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { return files.map((file) => { if (file.pubMagicMetadata?.data.editedTime) { diff --git a/web/apps/photos/src/services/upload/metadata.ts b/web/apps/photos/src/services/upload/metadata.ts index 4cda767a03..dcd83c3e3a 100644 --- a/web/apps/photos/src/services/upload/metadata.ts +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -8,6 +8,7 @@ import { tryToParseDateTime, validateAndGetCreationUnixTimeInMicroSeconds, } from "@ente/shared/time"; +import type { DataStream } from "@ente/shared/utils/data-stream"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload"; @@ -29,7 +30,6 @@ import { type ParsedMetadataJSON, } from "./takeout"; import { getFileName } from "./uploadService"; -import type { DataStream } from "@ente/shared/utils/data-stream"; const EXIF_TAGS_NEEDED = [ "DateTimeOriginal", diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b27843186f..9584b30ddc 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -7,6 +7,7 @@ import { CustomErrorMessage } from "@/next/types/ipc"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; +import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { @@ -59,7 +60,6 @@ import { } from "./thumbnail"; import uploadCancelService from "./uploadCancelService"; import UploadHttpClient from "./uploadHttpClient"; -import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream"; /** Upload files to cloud storage */ class UploadService { diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 03ef369826..29502213e3 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -441,6 +441,18 @@ export function isSharedFile(user: User, file: EnteFile) { return file.ownerID !== user.id; } +/** + * [Note: File name for local EnteFile objects] + * + * The title property in a file's metadata is the original file's name. The + * metadata of a file cannot be edited. So if later on the file's name is + * changed, then the edit is stored in the `editedName` property of the public + * metadata of the file. + * + * This function merges these edits onto the file object that we use locally. + * Effectively, post this step, the file's metadata.title can be used in lieu of + * its filename. + */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { return files.map((file) => { if (file.pubMagicMetadata?.data.editedTime) { From c486919547487f6bb63460b0809f4fc7bf03c276 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:06:11 +0530 Subject: [PATCH 75/82] Reuse --- web/apps/photos/src/components/PhotoFrame.tsx | 2 +- .../src/components/PhotoViewer/FileInfo/RenderFileName.tsx | 2 +- web/apps/photos/src/components/PhotoViewer/index.tsx | 2 +- web/apps/photos/src/components/PlaceholderThumbnails.tsx | 2 +- .../src/components/Search/SearchBar/searchInput/index.tsx | 2 +- .../photos/src/components/pages/gallery/PreviewCard.tsx | 2 +- web/apps/photos/src/constants/file.ts | 7 ------- web/apps/photos/src/constants/upload.ts | 2 +- web/apps/photos/src/services/clip-service.ts | 2 +- web/apps/photos/src/services/deduplicationService.ts | 2 +- web/apps/photos/src/services/download/index.ts | 2 +- web/apps/photos/src/services/export/index.ts | 2 +- web/apps/photos/src/services/export/migration.ts | 2 +- .../photos/src/services/machineLearning/mlWorkManager.ts | 2 +- .../photos/src/services/machineLearning/readerService.ts | 2 +- web/apps/photos/src/services/searchService.ts | 2 +- web/apps/photos/src/services/typeDetectionService.ts | 2 +- web/apps/photos/src/services/updateCreationTimeWithExif.ts | 2 +- web/apps/photos/src/services/upload/metadata.ts | 2 +- web/apps/photos/src/services/upload/thumbnail.ts | 2 +- web/apps/photos/src/services/upload/uploadManager.ts | 2 +- web/apps/photos/src/services/upload/uploadService.ts | 2 +- web/apps/photos/src/types/search/index.ts | 2 +- web/apps/photos/src/types/upload/index.ts | 2 +- web/apps/photos/src/utils/file/index.ts | 2 +- web/apps/photos/src/utils/machineLearning/index.ts | 2 +- web/apps/photos/src/utils/photoFrame/index.ts | 2 +- web/apps/photos/tests/upload.test.ts | 2 +- 28 files changed, 27 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/components/PhotoFrame.tsx b/web/apps/photos/src/components/PhotoFrame.tsx index 90e1cf32c3..5ff70b3d45 100644 --- a/web/apps/photos/src/components/PhotoFrame.tsx +++ b/web/apps/photos/src/components/PhotoFrame.tsx @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { PHOTOS_PAGES } from "@ente/shared/constants/pages"; import { CustomError } from "@ente/shared/error"; @@ -5,7 +6,6 @@ import useMemoSingleThreaded from "@ente/shared/hooks/useMemoSingleThreaded"; import { styled } from "@mui/material"; import PhotoViewer from "components/PhotoViewer"; import { TRASH_SECTION } from "constants/collection"; -import { FILE_TYPE } from "constants/file"; import { useRouter } from "next/router"; import { GalleryContext } from "pages/gallery"; import PhotoSwipe from "photoswipe"; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx index 1bee86c25a..c1b25e8506 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { FlexWrapper } from "@ente/shared/components/Container"; import PhotoOutlined from "@mui/icons-material/PhotoOutlined"; import VideocamOutlined from "@mui/icons-material/VideocamOutlined"; import Box from "@mui/material/Box"; -import { FILE_TYPE } from "constants/file"; import { useEffect, useState } from "react"; import { EnteFile } from "types/file"; import { makeHumanReadableStorage } from "utils/billing"; diff --git a/web/apps/photos/src/components/PhotoViewer/index.tsx b/web/apps/photos/src/components/PhotoViewer/index.tsx index 29da75e534..552c531f5d 100644 --- a/web/apps/photos/src/components/PhotoViewer/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/index.tsx @@ -16,6 +16,7 @@ import { isSupportedRawFormat, } from "utils/file"; +import { FILE_TYPE } from "@/media/file"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import AlbumOutlined from "@mui/icons-material/AlbumOutlined"; @@ -34,7 +35,6 @@ import InfoIcon from "@mui/icons-material/InfoOutlined"; import ReplayIcon from "@mui/icons-material/Replay"; import ZoomInOutlinedIcon from "@mui/icons-material/ZoomInOutlined"; import { Box, Button, styled } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; import { defaultLivePhotoDefaultOptions, photoSwipeV4Events, diff --git a/web/apps/photos/src/components/PlaceholderThumbnails.tsx b/web/apps/photos/src/components/PlaceholderThumbnails.tsx index caafbdce6f..7266bf1781 100644 --- a/web/apps/photos/src/components/PlaceholderThumbnails.tsx +++ b/web/apps/photos/src/components/PlaceholderThumbnails.tsx @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import { Overlay } from "@ente/shared/components/Container"; import PhotoOutlined from "@mui/icons-material/PhotoOutlined"; import PlayCircleOutlineOutlined from "@mui/icons-material/PlayCircleOutlineOutlined"; import { styled } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; interface Iprops { fileType: FILE_TYPE; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index d7cf151e66..87c45b59fd 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import CloseIcon from "@mui/icons-material/Close"; import { IconButton } from "@mui/material"; -import { FILE_TYPE } from "constants/file"; import { t } from "i18next"; import memoize from "memoize-one"; import pDebounce from "p-debounce"; diff --git a/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx b/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx index 8704258f89..b9288c59b2 100644 --- a/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx +++ b/web/apps/photos/src/components/pages/gallery/PreviewCard.tsx @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { Overlay } from "@ente/shared/components/Container"; import { CustomError } from "@ente/shared/error"; @@ -11,7 +12,6 @@ import { StaticThumbnail, } from "components/PlaceholderThumbnails"; import { TRASH_SECTION } from "constants/collection"; -import { FILE_TYPE } from "constants/file"; import { GAP_BTW_TILES, IMAGE_CONTAINER_MAX_WIDTH } from "constants/gallery"; import { DeduplicateContext } from "pages/deduplicate"; import { GalleryContext } from "pages/gallery"; diff --git a/web/apps/photos/src/constants/file.ts b/web/apps/photos/src/constants/file.ts index 46065136c9..f8d2a5ad8c 100644 --- a/web/apps/photos/src/constants/file.ts +++ b/web/apps/photos/src/constants/file.ts @@ -9,13 +9,6 @@ export const TYPE_HEIF = "heif"; export const TYPE_JPEG = "jpeg"; export const TYPE_JPG = "jpg"; -export enum FILE_TYPE { - IMAGE, - VIDEO, - LIVE_PHOTO, - OTHERS, -} - export const RAW_FORMATS = [ "heic", "rw2", diff --git a/web/apps/photos/src/constants/upload.ts b/web/apps/photos/src/constants/upload.ts index 1c677470ef..75918fae0b 100644 --- a/web/apps/photos/src/constants/upload.ts +++ b/web/apps/photos/src/constants/upload.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants"; -import { FILE_TYPE } from "constants/file"; import { FileTypeInfo, Location } from "types/upload"; // list of format that were missed by type-detection for some files. diff --git a/web/apps/photos/src/services/clip-service.ts b/web/apps/photos/src/services/clip-service.ts index eae9590fd5..ab1ce928c5 100644 --- a/web/apps/photos/src/services/clip-service.ts +++ b/web/apps/photos/src/services/clip-service.ts @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; -import { FILE_TYPE } from "constants/file"; import isElectron from "is-electron"; import PQueue from "p-queue"; import { Embedding } from "types/embedding"; diff --git a/web/apps/photos/src/services/deduplicationService.ts b/web/apps/photos/src/services/deduplicationService.ts index 9d8ab399ff..f9416396f2 100644 --- a/web/apps/photos/src/services/deduplicationService.ts +++ b/web/apps/photos/src/services/deduplicationService.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import HTTPService from "@ente/shared/network/HTTPService"; import { getEndpoint } from "@ente/shared/network/api"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { FILE_TYPE } from "constants/file"; import { EnteFile } from "types/file"; import { Metadata } from "types/upload"; import { hasFileHash } from "utils/upload"; diff --git a/web/apps/photos/src/services/download/index.ts b/web/apps/photos/src/services/download/index.ts index d2ad6b1f77..fe566700c8 100644 --- a/web/apps/photos/src/services/download/index.ts +++ b/web/apps/photos/src/services/download/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { openCache, type BlobCache } from "@/next/blob-cache"; import log from "@/next/log"; @@ -8,7 +9,6 @@ import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { isPlaybackPossible } from "@ente/shared/media/video-playback"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import isElectron from "is-electron"; import * as ffmpegService from "services/ffmpeg"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 0bf355d8d7..c949b46360 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; @@ -11,7 +12,6 @@ import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { CollectionExportNames, diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index a8c4e50689..6c13ce7fb3 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { wait } from "@ente/shared/utils"; -import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import downloadManager from "services/download"; import { getAllLocalFiles } from "services/fileService"; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index c5df14b224..a00c24ad3a 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { eventBus, Events } from "@ente/shared/events"; import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; -import { FILE_TYPE } from "constants/file"; import debounce from "debounce"; import PQueue from "p-queue"; import { JobResult } from "types/common/job"; diff --git a/web/apps/photos/src/services/machineLearning/readerService.ts b/web/apps/photos/src/services/machineLearning/readerService.ts index a18b3c9082..d453806412 100644 --- a/web/apps/photos/src/services/machineLearning/readerService.ts +++ b/web/apps/photos/src/services/machineLearning/readerService.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import { MLSyncContext, MLSyncFileContext } from "types/machineLearning"; import { getLocalFileImageBitmap, diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index dfe6f20068..362e76fbef 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import * as chrono from "chrono-node"; -import { FILE_TYPE } from "constants/file"; import { t } from "i18next"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; diff --git a/web/apps/photos/src/services/typeDetectionService.ts b/web/apps/photos/src/services/typeDetectionService.ts index 5b53eecbc0..598b100465 100644 --- a/web/apps/photos/src/services/typeDetectionService.ts +++ b/web/apps/photos/src/services/typeDetectionService.ts @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; import { CustomError } from "@ente/shared/error"; -import { FILE_TYPE } from "constants/file"; import { KNOWN_NON_MEDIA_FORMATS, WHITELISTED_FILE_FORMATS, diff --git a/web/apps/photos/src/services/updateCreationTimeWithExif.ts b/web/apps/photos/src/services/updateCreationTimeWithExif.ts index 667ae44f4e..e446219150 100644 --- a/web/apps/photos/src/services/updateCreationTimeWithExif.ts +++ b/web/apps/photos/src/services/updateCreationTimeWithExif.ts @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import type { FixOption } from "components/FixCreationTime"; -import { FILE_TYPE } from "constants/file"; import { getFileType } from "services/typeDetectionService"; import { EnteFile } from "types/file"; import { diff --git a/web/apps/photos/src/services/upload/metadata.ts b/web/apps/photos/src/services/upload/metadata.ts index dcd83c3e3a..d6acb9abd3 100644 --- a/web/apps/photos/src/services/upload/metadata.ts +++ b/web/apps/photos/src/services/upload/metadata.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { getFileNameSize } from "@/next/file"; import log from "@/next/log"; import { ElectronFile } from "@/next/types/file"; @@ -10,7 +11,6 @@ import { } from "@ente/shared/time"; import type { DataStream } from "@ente/shared/utils/data-stream"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import { FILE_READER_CHUNK_SIZE, NULL_LOCATION } from "constants/upload"; import * as ffmpegService from "services/ffmpeg"; import { getElectronFileStream, getFileStream } from "services/readerService"; diff --git a/web/apps/photos/src/services/upload/thumbnail.ts b/web/apps/photos/src/services/upload/thumbnail.ts index e2eccf9b89..d8ba3a2017 100644 --- a/web/apps/photos/src/services/upload/thumbnail.ts +++ b/web/apps/photos/src/services/upload/thumbnail.ts @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import { type Electron } from "@/next/types/ipc"; import { withTimeout } from "@ente/shared/utils"; -import { FILE_TYPE } from "constants/file"; import { BLACK_THUMBNAIL_BASE64 } from "constants/upload"; import * as ffmpeg from "services/ffmpeg"; import { heicToJPEG } from "services/heic-convert"; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 0c54d0f02d..3a99d35e13 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { potentialFileTypeFromExtension } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import { nameAndExtension } from "@/next/file"; @@ -11,7 +12,6 @@ import { Events, eventBus } from "@ente/shared/events"; import { wait } from "@ente/shared/utils"; import { Canceler } from "axios"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import { RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UPLOAD_RESULT, diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 9584b30ddc..d6132b300c 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { encodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import { basename } from "@/next/file"; @@ -9,7 +10,6 @@ import { EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; import { isDataStream, type DataStream } from "@ente/shared/utils/data-stream"; import { Remote } from "comlink"; -import { FILE_TYPE } from "constants/file"; import { FILE_CHUNKS_COMBINED_FOR_A_UPLOAD_PART, FILE_READER_CHUNK_SIZE, diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 29a1cffef2..5918bb9dde 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,4 +1,4 @@ -import { FILE_TYPE } from "constants/file"; +import { FILE_TYPE } from "@/media/file"; import { City } from "services/locationSearchService"; import { LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index c687911c71..bdcef330b2 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -1,10 +1,10 @@ +import { FILE_TYPE } from "@/media/file"; import type { ElectronFile } from "@/next/types/file"; import { B64EncryptionResult, LocalFileAttributes, } from "@ente/shared/crypto/types"; import type { DataStream } from "@ente/shared/utils/data-stream"; -import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { FilePublicMagicMetadata, diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 29502213e3..9cc74373e7 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; @@ -7,7 +8,6 @@ import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils"; import { - FILE_TYPE, RAW_FORMATS, SUPPORTED_RAW_FORMATS, TYPE_HEIC, diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index a89bccc4ca..4e9b795ec1 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -1,6 +1,6 @@ +import { FILE_TYPE } from "@/media/file"; import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import PQueue from "p-queue"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; diff --git a/web/apps/photos/src/utils/photoFrame/index.ts b/web/apps/photos/src/utils/photoFrame/index.ts index faf0679e7f..ccd47b604d 100644 --- a/web/apps/photos/src/utils/photoFrame/index.ts +++ b/web/apps/photos/src/utils/photoFrame/index.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; -import { FILE_TYPE } from "constants/file"; import { LivePhotoSourceURL, SourceURLs } from "services/download"; import { EnteFile } from "types/file"; import { SetSelectedState } from "types/gallery"; diff --git a/web/apps/photos/tests/upload.test.ts b/web/apps/photos/tests/upload.test.ts index 5a05bb991d..3fc7e8541e 100644 --- a/web/apps/photos/tests/upload.test.ts +++ b/web/apps/photos/tests/upload.test.ts @@ -1,5 +1,5 @@ +import { FILE_TYPE } from "@/media/file"; import { tryToParseDateTime } from "@ente/shared/time"; -import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import { getLocalFiles } from "services/fileService"; import { From e919dfd09d0c2580c014e2bda47d928019613642 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:07:19 +0530 Subject: [PATCH 76/82] Scope --- .../src/components/EnteDateTimePicker.tsx | 7 ++-- .../PhotoViewer/FileInfo/RenderCaption.tsx | 3 +- web/apps/photos/src/constants/file.ts | 36 ----------------- web/apps/photos/src/utils/file/index.ts | 40 +++++++++++++++---- 4 files changed, 37 insertions(+), 49 deletions(-) delete mode 100644 web/apps/photos/src/constants/file.ts diff --git a/web/apps/photos/src/components/EnteDateTimePicker.tsx b/web/apps/photos/src/components/EnteDateTimePicker.tsx index ee5426ebcc..e53ed65b98 100644 --- a/web/apps/photos/src/components/EnteDateTimePicker.tsx +++ b/web/apps/photos/src/components/EnteDateTimePicker.tsx @@ -5,10 +5,9 @@ import { MobileDateTimePicker, } from "@mui/x-date-pickers"; import { AdapterDateFns } from "@mui/x-date-pickers/AdapterDateFns"; -import { - MAX_EDITED_CREATION_TIME, - MIN_EDITED_CREATION_TIME, -} from "constants/file"; + +const MIN_EDITED_CREATION_TIME = new Date(1800, 0, 1); +const MAX_EDITED_CREATION_TIME = new Date(); interface Props { initialValue?: Date; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx index 871da2b05f..3a5dbb6bc2 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCaption.tsx @@ -3,7 +3,6 @@ import { FlexWrapper } from "@ente/shared/components/Container"; import Close from "@mui/icons-material/Close"; import Done from "@mui/icons-material/Done"; import { Box, IconButton, TextField } from "@mui/material"; -import { MAX_CAPTION_SIZE } from "constants/file"; import { Formik } from "formik"; import { t } from "i18next"; import { useState } from "react"; @@ -12,6 +11,8 @@ import { changeCaption, updateExistingFilePubMetadata } from "utils/file"; import * as Yup from "yup"; import { SmallLoadingSpinner } from "../styledComponents/SmallLoadingSpinner"; +export const MAX_CAPTION_SIZE = 5000; + interface formValues { caption: string; } diff --git a/web/apps/photos/src/constants/file.ts b/web/apps/photos/src/constants/file.ts deleted file mode 100644 index f8d2a5ad8c..0000000000 --- a/web/apps/photos/src/constants/file.ts +++ /dev/null @@ -1,36 +0,0 @@ -export const MIN_EDITED_CREATION_TIME = new Date(1800, 0, 1); -export const MAX_EDITED_CREATION_TIME = new Date(); - -export const MAX_EDITED_FILE_NAME_LENGTH = 100; -export const MAX_CAPTION_SIZE = 5000; - -export const TYPE_HEIC = "heic"; -export const TYPE_HEIF = "heif"; -export const TYPE_JPEG = "jpeg"; -export const TYPE_JPG = "jpg"; - -export const RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "raf", - "nef", - "psd", - "dng", - "tif", -]; -export const SUPPORTED_RAW_FORMATS = [ - "heic", - "rw2", - "tiff", - "arw", - "cr3", - "cr2", - "nef", - "psd", - "dng", - "tif", -]; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 9cc74373e7..ac82eaf6e8 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -7,14 +7,6 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils"; -import { - RAW_FORMATS, - SUPPORTED_RAW_FORMATS, - TYPE_HEIC, - TYPE_HEIF, - TYPE_JPEG, - TYPE_JPG, -} from "constants/file"; import { t } from "i18next"; import isElectron from "is-electron"; import { moveToHiddenCollection } from "services/collectionService"; @@ -48,6 +40,38 @@ import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; import { safeFileName } from "utils/native-fs"; import { writeStream } from "utils/native-stream"; +const TYPE_HEIC = "heic"; +const TYPE_HEIF = "heif"; +const TYPE_JPEG = "jpeg"; +const TYPE_JPG = "jpg"; + +const RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "raf", + "nef", + "psd", + "dng", + "tif", +]; + +const SUPPORTED_RAW_FORMATS = [ + "heic", + "rw2", + "tiff", + "arw", + "cr3", + "cr2", + "nef", + "psd", + "dng", + "tif", +]; + export enum FILE_OPS_TYPE { DOWNLOAD, FIX_TIME, From 9de8a3d40a1a1ae6c5c387c998eb5be96aff9544 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:19:37 +0530 Subject: [PATCH 77/82] Fixes --- web/apps/cast/src/pages/slideshow.tsx | 2 +- .../cast/src/services/castDownloadManager.ts | 2 +- web/apps/photos/src/services/ffmpeg.ts | 19 +++++-------------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/web/apps/cast/src/pages/slideshow.tsx b/web/apps/cast/src/pages/slideshow.tsx index 000d18380b..d13a263464 100644 --- a/web/apps/cast/src/pages/slideshow.tsx +++ b/web/apps/cast/src/pages/slideshow.tsx @@ -1,7 +1,7 @@ +import { FILE_TYPE } from "@/media/file"; import log from "@/next/log"; import PairedSuccessfullyOverlay from "components/PairedSuccessfullyOverlay"; import { PhotoAuditorium } from "components/PhotoAuditorium"; -import { FILE_TYPE } from "@/media/file"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { diff --git a/web/apps/cast/src/services/castDownloadManager.ts b/web/apps/cast/src/services/castDownloadManager.ts index 1b890fbbc1..40a5f25af1 100644 --- a/web/apps/cast/src/services/castDownloadManager.ts +++ b/web/apps/cast/src/services/castDownloadManager.ts @@ -1,8 +1,8 @@ +import { FILE_TYPE } from "@/media/file"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getCastFileURL } from "@ente/shared/network/api"; -import { FILE_TYPE } from "@/media/file"; import { EnteFile } from "types/file"; import { generateStreamFromArrayBuffer } from "utils/file"; diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index 58b182e73b..f6f4b017d1 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -105,9 +105,7 @@ export async function extractVideoMetadata(file: File | ElectronFile) { file, `metadata.txt`, ); - return parseFFmpegExtractedMetadata( - new Uint8Array(await metadata.arrayBuffer()), - ); + return parseFFmpegExtractedMetadata(metadata); } enum MetadataTags { @@ -236,9 +234,10 @@ const ffmpegExec2 = async ( inputFile: File | ElectronFile, outputFileName: string, timeoutMS: number = 0, -): Promise => { +) => { const electron = globalThis.electron; if (electron || false) { + throw new Error("WIP"); // return electron.ffmpegExec( // command, // /* TODO(MR): ElectronFile changes */ @@ -247,16 +246,8 @@ const ffmpegExec2 = async ( // timeoutMS, // ); } else { - return workerFactory - .lazy() - .then((worker) => - worker.execute( - command, - /* TODO(MR): ElectronFile changes */ inputFile as File, - outputFileName, - timeoutMS, - ), - ); + /* TODO(MR): ElectronFile changes */ + return ffmpegExecWeb(command, inputFile as File, timeoutMS); } }; From a08df9a83999a52c39052425e64f8c6978623305 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:36:04 +0530 Subject: [PATCH 78/82] Clarification after asking in Discord --- web/apps/photos/src/services/upload/uploadService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index d6132b300c..e208fd4447 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -757,6 +757,10 @@ const areFilesSameHash = (f: Metadata, g: Metadata) => { } }; +/** + * Older files that were uploaded before we introduced hashing will not have + * hashes, so retain and use the logic we used back then for such files. + */ const areFilesSameNoHash = (f: Metadata, g: Metadata) => { /* * The maximum difference in the creation/modification times of two similar From 7dba4c0af4bc3935971a57d7738e4c193383f77d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:41:35 +0530 Subject: [PATCH 79/82] Fix --- web/apps/photos/src/services/download/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/download/index.ts b/web/apps/photos/src/services/download/index.ts index fe566700c8..7a2c359329 100644 --- a/web/apps/photos/src/services/download/index.ts +++ b/web/apps/photos/src/services/download/index.ts @@ -617,7 +617,7 @@ async function getPlayableVideo( new File([videoBlob], videoNameTitle), ); log.info(`video successfully converted ${videoNameTitle}`); - return new Blob([await mp4ConvertedVideo.arrayBuffer()]); + return new Blob([mp4ConvertedVideo]); } } catch (e) { log.error("video conversion failed", e); From 5f146aa597d48ae1f18f0da2165092de627674c5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 13:45:56 +0530 Subject: [PATCH 80/82] Fix --- web/apps/photos/src/utils/file/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index ac82eaf6e8..fa681c2cde 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -333,7 +333,7 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } } - if (!isFileHEIC(exactType)) { + if (isFileHEIC(exactType)) { // If it is an HEIC file, use our web HEIC converter. return await heicToJPEG(imageBlob); } From f32a396b36d8f8f7c964b092ac1e0cb949871d3c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 14:07:41 +0530 Subject: [PATCH 81/82] Fix video thumbnailing (the .jpeg extension is required) --- desktop/src/main/ipc.ts | 3 +- desktop/src/main/services/ffmpeg.ts | 3 +- desktop/src/main/services/image.ts | 4 +- desktop/src/main/utils-temp.ts | 9 ++-- desktop/src/preload.ts | 9 +++- web/apps/photos/src/services/ffmpeg.ts | 50 ++++++++++++--------- web/apps/photos/src/worker/ffmpeg.worker.ts | 22 ++++++--- web/packages/next/types/ipc.ts | 7 +++ 8 files changed, 71 insertions(+), 36 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 91efbb09f0..15d51530df 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -161,8 +161,9 @@ export const attachIPCHandlers = () => { _, command: string[], dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, - ) => ffmpegExec(command, dataOrPath, timeoutMS), + ) => ffmpegExec(command, dataOrPath, outputFileExtension, timeoutMS), ); // - ML diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index f99f7ef8f1..1505d8a96d 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -40,6 +40,7 @@ const outputPathPlaceholder = "OUTPUT"; export const ffmpegExec = async ( command: string[], dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ): Promise => { // TODO (MR): This currently copies files for both input and output. This @@ -56,7 +57,7 @@ export const ffmpegExec = async ( isInputFileTemporary = false; } - const outputFilePath = await makeTempFilePath(); + const outputFilePath = await makeTempFilePath(outputFileExtension); try { if (dataOrPath instanceof Uint8Array) await fs.writeFile(inputFilePath, dataOrPath); diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts index 7fae507578..d8108c635f 100644 --- a/desktop/src/main/services/image.ts +++ b/desktop/src/main/services/image.ts @@ -9,7 +9,7 @@ import { deleteTempFile, makeTempFilePath } from "../utils-temp"; export const convertToJPEG = async (imageData: Uint8Array) => { const inputFilePath = await makeTempFilePath(); - const outputFilePath = await makeTempFilePath(".jpeg"); + const outputFilePath = await makeTempFilePath("jpeg"); // Construct the command first, it may throw NotAvailable on win32. const command = convertToJPEGCommand(inputFilePath, outputFilePath); @@ -77,7 +77,7 @@ export const generateImageThumbnail = async ( isInputFileTemporary = false; } - const outputFilePath = await makeTempFilePath(".jpeg"); + const outputFilePath = await makeTempFilePath("jpeg"); // Construct the command first, it may throw `NotAvailable` on win32. let quality = 70; diff --git a/desktop/src/main/utils-temp.ts b/desktop/src/main/utils-temp.ts index f48b2c388f..a52daf619d 100644 --- a/desktop/src/main/utils-temp.ts +++ b/desktop/src/main/utils-temp.ts @@ -29,17 +29,18 @@ const randomPrefix = () => { * * The function returns the path to a file in the system temp directory (in an * Ente specific folder therin) with a random prefix and an (optional) - * {@link suffix}. + * {@link extension}. * - * It ensures that there is no existing file with the same name already. + * It ensures that there is no existing item with the same name already. * * Use {@link deleteTempFile} to remove this file when you're done. */ -export const makeTempFilePath = async (suffix?: string) => { +export const makeTempFilePath = async (extension?: string) => { const tempDir = await enteTempDirPath(); + const suffix = extension ? "." + extension : ""; let result: string; do { - result = path.join(tempDir, `${randomPrefix()}${suffix ?? ""}`); + result = path.join(tempDir, randomPrefix() + suffix); } while (existsSync(result)); return result; }; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 728e8d012b..ea3cf1e054 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -142,9 +142,16 @@ const generateImageThumbnail = ( const ffmpegExec = ( command: string[], dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ): Promise => - ipcRenderer.invoke("ffmpegExec", command, dataOrPath, timeoutMS); + ipcRenderer.invoke( + "ffmpegExec", + command, + dataOrPath, + outputFileExtension, + timeoutMS, + ); // - ML diff --git a/web/apps/photos/src/services/ffmpeg.ts b/web/apps/photos/src/services/ffmpeg.ts index f6f4b017d1..b1436f17b5 100644 --- a/web/apps/photos/src/services/ffmpeg.ts +++ b/web/apps/photos/src/services/ffmpeg.ts @@ -25,10 +25,14 @@ import { type DedicatedFFmpegWorker } from "worker/ffmpeg.worker"; * * See also {@link generateVideoThumbnailNative}. */ -export const generateVideoThumbnailWeb = async (blob: Blob) => { - const thumbnailAtTime = (seekTime: number) => - ffmpegExecWeb(commandForThumbnailAtTime(seekTime), blob, 0); +export const generateVideoThumbnailWeb = async (blob: Blob) => + generateVideoThumbnail((seekTime: number) => + ffmpegExecWeb(genThumbnailCommand(seekTime), blob, "jpeg", 0), + ); +const generateVideoThumbnail = async ( + thumbnailAtTime: (seekTime: number) => Promise, +) => { try { // Try generating thumbnail at seekTime 1 second. return await thumbnailAtTime(1); @@ -56,21 +60,17 @@ export const generateVideoThumbnailWeb = async (blob: Blob) => { export const generateVideoThumbnailNative = async ( electron: Electron, dataOrPath: Uint8Array | string, -) => { - const thumbnailAtTime = (seekTime: number) => - electron.ffmpegExec(commandForThumbnailAtTime(seekTime), dataOrPath, 0); +) => + generateVideoThumbnail((seekTime: number) => + electron.ffmpegExec( + genThumbnailCommand(seekTime), + dataOrPath, + "jpeg", + 0, + ), + ); - try { - // Try generating thumbnail at seekTime 1 second. - return await thumbnailAtTime(1); - } catch (e) { - // If that fails, try again at the beginning. If even this throws, let - // it fail. - return await thumbnailAtTime(0); - } -}; - -const commandForThumbnailAtTime = (seekTime: number) => [ +const genThumbnailCommand = (seekTime: number) => [ ffmpegPathPlaceholder, "-i", inputPathPlaceholder, @@ -103,7 +103,7 @@ export async function extractVideoMetadata(file: File | ElectronFile) { outputPathPlaceholder, ], file, - `metadata.txt`, + "txt", ); return parseFFmpegExtractedMetadata(metadata); } @@ -184,7 +184,7 @@ export async function convertToMP4(file: File) { outputPathPlaceholder, ], file, - "output.mp4", + "mp4", 30 * 1000, ); } @@ -198,10 +198,11 @@ export async function convertToMP4(file: File) { const ffmpegExecWeb = async ( command: string[], blob: Blob, + outputFileExtension: string, timeoutMs: number, ) => { const worker = await workerFactory.lazy(); - return await worker.exec(command, blob, timeoutMs); + return await worker.exec(command, blob, outputFileExtension, timeoutMs); }; /** @@ -232,7 +233,7 @@ const ffmpegExecNative = async ( const ffmpegExec2 = async ( command: string[], inputFile: File | ElectronFile, - outputFileName: string, + outputFileExtension: string, timeoutMS: number = 0, ) => { const electron = globalThis.electron; @@ -247,7 +248,12 @@ const ffmpegExec2 = async ( // ); } else { /* TODO(MR): ElectronFile changes */ - return ffmpegExecWeb(command, inputFile as File, timeoutMS); + return ffmpegExecWeb( + command, + inputFile as File, + outputFileExtension, + timeoutMS, + ); } }; diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index b30b2fa38f..a9f2ad56bc 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -26,10 +26,16 @@ export class DedicatedFFmpegWorker { * This is a sibling of {@link ffmpegExec} exposed by the desktop app in * `ipc.ts`. See [Note: FFmpeg in Electron]. */ - async exec(command: string[], blob: Blob, timeoutMs): Promise { + async exec( + command: string[], + blob: Blob, + outputFileExtension: string, + timeoutMs, + ): Promise { if (!this.ffmpeg.isLoaded()) await this.ffmpeg.load(); - const go = () => ffmpegExec(this.ffmpeg, command, blob); + const go = () => + ffmpegExec(this.ffmpeg, command, outputFileExtension, blob); const request = this.ffmpegTaskQueue.queueUpRequest(() => timeoutMs ? withTimeout(go(), timeoutMs) : go(), @@ -41,9 +47,15 @@ export class DedicatedFFmpegWorker { expose(DedicatedFFmpegWorker, self); -const ffmpegExec = async (ffmpeg: FFmpeg, command: string[], blob: Blob) => { - const inputPath = `${randomPrefix()}.in`; - const outputPath = `${randomPrefix()}.out`; +const ffmpegExec = async ( + ffmpeg: FFmpeg, + command: string[], + outputFileExtension: string, + blob: Blob, +) => { + const inputPath = randomPrefix(); + const outputSuffix = outputFileExtension ? "." + outputFileExtension : ""; + const outputPath = randomPrefix() + outputSuffix; const cmd = substitutePlaceholders(command, inputPath, outputPath); diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index d392b6f3b4..cdc3597ec9 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -254,6 +254,12 @@ export interface Electron { * a temporary file, and then that path gets substituted in the FFmpeg * {@link command} in lieu of {@link inputPathPlaceholder}. * + * @param outputFileExtension The extension (without the dot, e.g. "jpeg") + * to use for the output file that we ask FFmpeg to create in + * {@param command}. While this file will eventually get deleted, and we'll + * just return its contents, for some FFmpeg command the extension matters + * (e.g. conversion to a JPEG fails if the extension is arbitrary). + * * @param timeoutMS If non-zero, then abort and throw a timeout error if the * ffmpeg command takes more than the given number of milliseconds. * @@ -263,6 +269,7 @@ export interface Electron { ffmpegExec: ( command: string[], dataOrPath: Uint8Array | string, + outputFileExtension: string, timeoutMS: number, ) => Promise; From 80802d44e360d31d7f6cf740b49df8413fc36252 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 24 Apr 2024 14:49:29 +0530 Subject: [PATCH 82/82] Better log --- web/apps/photos/src/worker/ffmpeg.worker.ts | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/web/apps/photos/src/worker/ffmpeg.worker.ts b/web/apps/photos/src/worker/ffmpeg.worker.ts index a9f2ad56bc..03893efba6 100644 --- a/web/apps/photos/src/worker/ffmpeg.worker.ts +++ b/web/apps/photos/src/worker/ffmpeg.worker.ts @@ -64,7 +64,7 @@ const ffmpegExec = async ( try { ffmpeg.FS("writeFile", inputPath, inputData); - log.info(`Running FFmpeg (wasm) command ${cmd}`); + log.debug(() => `[wasm] ffmpeg ${cmd.join(" ")}`); await ffmpeg.run(...cmd); return ffmpeg.FS("readFile", outputPath); @@ -98,14 +98,16 @@ const substitutePlaceholders = ( inputFilePath: string, outputFilePath: string, ) => - command.map((segment) => { - if (segment == ffmpegPathPlaceholder) { - return ""; - } else if (segment == inputPathPlaceholder) { - return inputFilePath; - } else if (segment == outputPathPlaceholder) { - return outputFilePath; - } else { - return segment; - } - }); + command + .map((segment) => { + if (segment == ffmpegPathPlaceholder) { + return undefined; + } else if (segment == inputPathPlaceholder) { + return inputFilePath; + } else if (segment == outputPathPlaceholder) { + return outputFilePath; + } else { + return segment; + } + }) + .filter((c) => !!c);