From a286b11adba1e01967a2ef3fcebf87b7b7cf3157 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 23 Apr 2024 12:55:27 +0530 Subject: [PATCH] Checkpoint --- .../src/services/upload/uploadService.ts | 104 +++++++++--------- web/apps/photos/src/utils/native-stream.ts | 18 ++- 2 files changed, 66 insertions(+), 56 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index c0ec389743..f3e8b852e9 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -54,12 +54,9 @@ import { getNonEmptyMagicMetadataProps, updateMagicMetadata, } from "utils/magicMetadata"; +import { readStream } from "utils/native-stream"; import { findMatchingExistingFiles } from "utils/upload"; -import { - getElectronFileStream, - getFileStream, - getUint8ArrayView, -} from "../readerService"; +import { getFileStream, getUint8ArrayView } from "../readerService"; import { getFileType } from "../typeDetectionService"; import { MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT, @@ -431,6 +428,29 @@ const readFileOrPath = async ( ): Promise => { log.info(`Reading file ${fopLabel(fileOrPath)} `); + let dataOrStream: Uint8Array | DataStream; + if (fileOrPath instanceof File) { + const file = fileOrPath; + if (file.size > MULTIPART_PART_SIZE) { + dataOrStream = getFileStream(file, FILE_READER_CHUNK_SIZE); + } else { + dataOrStream = new Uint8Array(await file.arrayBuffer()); + } + } else { + const path = fileOrPath; + const { stream, size } = await readStream(path); + if (size > MULTIPART_PART_SIZE) { + const chunkCount = Math.ceil(size / FILE_READER_CHUNK_SIZE); + dataOrStream = { stream, chunkCount }; + } else { + dataOrStream = new Uint8Array( + await new Response(stream).arrayBuffer(), + ); + } + } + + let filedata: Uint8Array | DataStream; + // If it's a file, read-in its data. We need to do it once anyway for // generating the thumbnail. const dataOrPath = @@ -438,55 +458,39 @@ const readFileOrPath = async ( ? new Uint8Array(await fileOrPath.arrayBuffer()) : fileOrPath; - let thumbnail: Uint8Array; + // let thumbnail: Uint8Array; - const electron = globalThis.electron; - if (electron) { - if !moduleState.isNativeImageThumbnailCreationNotAvailable; - try { - return await generateImageThumbnailNative(electron, fileOrPath); - } catch (e) { - if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isNativeThumbnailCreationNotAvailable = true; - } else { - log.error("Native thumbnail creation failed", e); - } - } - } + // const electron = globalThis.electron; + // if (electron) { + // if !moduleState.isNativeImageThumbnailCreationNotAvailable; + // try { + // return await generateImageThumbnailNative(electron, fileOrPath); + // } catch (e) { + // if (e.message == CustomErrorMessage.NotAvailable) { + // moduleState.isNativeThumbnailCreationNotAvailable = true; + // } else { + // log.error("Native thumbnail creation failed", e); + // } + // } + // } - let filedata: Uint8Array | DataStream; - if (!(rawFile instanceof File)) { - if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = await getElectronFileStream( - rawFile, - FILE_READER_CHUNK_SIZE, - ); - } else { - filedata = await getUint8ArrayView(rawFile); - } - } else if (rawFile.size > MULTIPART_PART_SIZE) { - filedata = getFileStream(rawFile, FILE_READER_CHUNK_SIZE); - } else { - filedata = await getUint8ArrayView(rawFile); - } + // try { + // const thumbnail = + // fileTypeInfo.fileType === FILE_TYPE.IMAGE + // ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) + // : await generateVideoThumbnail(blob); - try { - const thumbnail = - fileTypeInfo.fileType === FILE_TYPE.IMAGE - ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo) - : await generateVideoThumbnail(blob); + // if (thumbnail.length == 0) throw new Error("Empty thumbnail"); + // return { thumbnail, hasStaticThumbnail: false }; + // } catch (e) { + // log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); + // return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; + // } - if (thumbnail.length == 0) throw new Error("Empty thumbnail"); - return { thumbnail, hasStaticThumbnail: false }; - } catch (e) { - log.error(`Failed to generate ${fileTypeInfo.exactType} thumbnail`, e); - return { thumbnail: fallbackThumbnail(), hasStaticThumbnail: true }; - } - - if (filedata instanceof Uint8Array) { - } else { - filedata.stream; - } + // if (filedata instanceof Uint8Array) { + // } else { + // filedata.stream; + // } log.info(`read file data successfully ${getFileNameSize(rawFile)} `); diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index b98e6ae1a2..aef06845c0 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -14,13 +14,13 @@ * @param path The path on the file on the user's local filesystem whose * contents we want to stream. * - * @return A standard web {@link Response} object that contains the contents of - * the file. In particular, `response.body` will be a {@link ReadableStream} - * that can be used to read the files contents in a streaming, chunked, manner. - * Also, the response is guaranteed to have a "Content-Length" header indicating + * @return A (ReadableStream, size) tuple. The {@link ReadableStream} can be + * used to read the files contents in a streaming manner. The size value is the * the size of the file that we'll be reading from disk. */ -export const readStream = async (path: string) => { +export const readStream = async ( + path: string, +): Promise<{ stream: ReadableStream; size: number }> => { const req = new Request(`stream://read${path}`, { method: "GET", }); @@ -31,7 +31,13 @@ export const readStream = async (path: string) => { `Failed to read stream from ${path}: HTTP ${res.status}`, ); - return res; + const size = +res.headers["Content-Length"]; + if (isNaN(size)) + throw new Error( + `Got a numeric Content-Length when reading a stream. The response was ${res}`, + ); + + return { stream: res.body, size }; }; /**