TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] Rearrange upload code - Part 2/2 (#5786)

Browse Source 
Fix most (but not all) of the temporary escape hatches added during
https://github.com/ente-io/ente/pull/5779.
This commit is contained in:
Manav Rathi
2025-05-02 20:04:17 +05:30
committed by GitHub
parent 69c6adcd06 11aba9df96
commit 48d9d03b32
7 changed files with 182 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
web/apps/photos/src/services/upload-manager.ts
View File

@@ -22,8 +22,8 @@ import {
} from "ente-gallery/services/upload/takeout";
import UploadService, {
areLivePhotoAssets,
upload,
uploadItemFileName,
uploader,
type PotentialLivePhotoAsset,
type UploadAsset,
} from "ente-gallery/services/upload/upload-service";
@@ -557,7 +557,7 @@ class UploadManager {
uiService.setFileProgress(localID, 0);
await wait(0);
const { uploadResult, uploadedFile } = await uploader(
const { uploadResult, uploadedFile } = await upload(
uploadableItem,
this.uploaderName,
this.existingFiles,
@@ -735,7 +735,7 @@ export const uploadManager = new UploadManager();
*
* - On to the {@link ClusteredUploadItem} we attach the corresponding
* {@link collection}, giving us {@link UploadableUploadItem}. This is what
* gets queued and then passed to the {@link uploader}.
* gets queued and then passed to the {@link upload}.
*/
type UploadItemWithCollectionIDAndName = UploadAsset & {
/** A unique ID for the duration of the upload */

39
web/packages/gallery/services/upload/date.ts
View File

@@ -1,6 +1,3 @@
// TODO: Audit this file
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { nameAndExtension } from "ente-base/file-name";
import log from "ente-base/log";
@@ -27,10 +24,10 @@ export const tryParseEpochMicrosecondsFromFileName = (
};
// Not sure why we have a try catch, but until there is a chance to validate
// that it doesn't indeed throw, move out the actual logic into this separate
// and more readable function.
// that it doesn't indeed throw, keep this actual logic into this separate and
// more readable function.
const parseEpochMicrosecondsFromFileName = (fileName: string) => {
let date: Date;
let date: Date | undefined;
fileName = fileName.trim();
@@ -38,15 +35,16 @@ const parseEpochMicrosecondsFromFileName = (fileName: string) => {
if (fileName.startsWith("IMG-") || fileName.startsWith("VID-")) {
// WhatsApp media files
// - e.g. "IMG-20171218-WA0028.jpg"
// @ts-ignore
date = parseDateFromFusedDateString(fileName.split("-")[1]);
const p = fileName.split("-");
const dateString = p[1];
if (dateString) {
date = parseDateFromFusedDateString(dateString);
}
} else if (fileName.startsWith("Screenshot_")) {
// Screenshots on Android
// - e.g. "Screenshot_20181227-152914.jpg"
// @ts-ignore
date = parseDateFromFusedDateString(
fileName.replaceAll("Screenshot_", ""),
);
const dateString = fileName.replace("Screenshot_", "");
date = parseDateFromFusedDateString(dateString);
} else if (fileName.startsWith("signal-")) {
// Signal images
//
@@ -63,16 +61,13 @@ const parseEpochMicrosecondsFromFileName = (fileName: string) => {
const p = fileName.split("-");
if (p.length > 5) {
const dateString = `${p[1]}${p[2]}${p[3]}-${p[4]}${p[5]}${p[6]}`;
// @ts-ignore
date = parseDateFromFusedDateString(dateString);
} else {
const dateString = `${p[1]}${p[2]}${p[3]}-${p[4]}`;
// @ts-ignore
} else if (p.length > 1) {
const dateString = `${p[1]}${p[2] ?? ""}${p[3] ?? ""}-${p[4] ?? ""}`;
date = parseDateFromFusedDateString(dateString);
}
}
// @ts-ignore
if (!date) {
const [name] = nameAndExtension(fileName);
@@ -97,16 +92,13 @@ const parseEpochMicrosecondsFromFileName = (fileName: string) => {
const p = name.split("_");
if (p.length == 3) {
const dateString = `${p[0]}-${p[1]}`;
// @ts-ignore
date = parseDateFromFusedDateString(dateString);
}
}
}
// Generic pattern.
// @ts-ignore
if (!date) {
// @ts-ignore
date = parseDateFromDigitGroups(fileName);
}
@@ -131,8 +123,6 @@ const parseEpochMicrosecondsFromFileName = (fileName: string) => {
}
};
const currentYear = new Date().getFullYear();
/**
* An intermediate data structure we use for the functions in this file. It
* stores the various components of a JavaScript date.
@@ -206,7 +196,10 @@ const validateAndGetDateFromComponents = (components: DateComponents) => {
if (!isDatePartValid(date, components)) {
return undefined;
}
if (date.getFullYear() < 1990 || date.getFullYear() > currentYear + 1) {
if (
date.getFullYear() < 1990 ||
date.getFullYear() > new Date().getFullYear() + 1
) {
return undefined;
}
return date;

0
web/packages/gallery/services/upload.ts → web/packages/gallery/services/upload/index.ts
View File

135
web/packages/gallery/services/upload/remote.ts
View File

@@ -1,7 +1,7 @@
// TODO: Audit this file
/* eslint-disable */
// @ts-nocheck
/* eslint-disable @typescript-eslint/ban-ts-comment */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
import {
authenticatedPublicAlbumsRequestHeaders,
authenticatedRequestHeaders,
@@ -14,7 +14,6 @@ import { apiURL, uploaderOrigin } from "ente-base/origins";
import { type EnteFile } from "ente-media/file";
import { CustomError, handleUploadError } from "ente-shared/error";
import HTTPService from "ente-shared/network/HTTPService";
import { getToken } from "ente-shared/storage/localStorage/helpers";
import { z } from "zod";
import type { MultipartUploadURLs, UploadFile } from "./upload-service";
@@ -32,19 +31,24 @@ export type ObjectUploadURL = z.infer<typeof ObjectUploadURL>;
const ObjectUploadURLResponse = z.object({ urls: ObjectUploadURL.array() });
export class PhotosUploadHttpClient {
/**
* Lowest layer for file upload related HTTP operations when we're running in
* the context of the photos app.
*/
export class PhotosUploadHTTPClient {
async uploadFile(uploadFile: UploadFile): Promise<EnteFile> {
try {
const token = getToken();
if (!token) {
return;
}
const url = await apiURL("/files");
const headers = await authenticatedRequestHeaders();
const response = await retryAsyncOperation(
() =>
HTTPService.post(url, uploadFile, null, {
"X-Auth-Token": token,
}),
HTTPService.post(
url,
uploadFile,
// @ts-ignore
null,
headers,
),
handleUploadError,
);
return response.data;
@@ -72,26 +76,17 @@ export class PhotosUploadHttpClient {
headers: await authenticatedRequestHeaders(),
});
ensureOk(res);
return (
// TODO: The as cast will not be needed when tsc strict mode is
// enabled for this code.
ObjectUploadURLResponse.parse(await res.json())
.urls as ObjectUploadURL[]
);
return ObjectUploadURLResponse.parse(await res.json()).urls;
}
async fetchMultipartUploadURLs(
count: number,
): Promise<MultipartUploadURLs> {
try {
const token = getToken();
if (!token) {
return;
}
const response = await HTTPService.get(
await apiURL("/files/multipart-upload-urls"),
{ count },
{ "X-Auth-Token": token },
await authenticatedRequestHeaders(),
);
return response.data.urls;
@@ -104,7 +99,7 @@ export class PhotosUploadHttpClient {
async putFile(
fileUploadURL: ObjectUploadURL,
file: Uint8Array,
progressTracker,
progressTracker: unknown,
): Promise<string> {
try {
await retryAsyncOperation(
@@ -112,6 +107,7 @@ export class PhotosUploadHttpClient {
HTTPService.put(
fileUploadURL.url,
file,
// @ts-ignore
null,
null,
progressTracker,
@@ -120,7 +116,12 @@ export class PhotosUploadHttpClient {
);
return fileUploadURL.objectKey;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
if (
!(
e instanceof Error &&
e.message == CustomError.UPLOAD_CANCELLED
)
) {
log.error("putFile to dataStore failed ", e);
}
throw e;
@@ -130,7 +131,7 @@ export class PhotosUploadHttpClient {
async putFileV2(
fileUploadURL: ObjectUploadURL,
file: Uint8Array,
progressTracker,
progressTracker: unknown,
): Promise<string> {
try {
const origin = await uploaderOrigin();
@@ -138,6 +139,7 @@ export class PhotosUploadHttpClient {
HTTPService.put(
`${origin}/file-upload`,
file,
// @ts-ignore
null,
{ "UPLOAD-URL": fileUploadURL.url },
progressTracker,
@@ -145,7 +147,12 @@ export class PhotosUploadHttpClient {
);
return fileUploadURL.objectKey;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
if (
!(
e instanceof Error &&
e.message == CustomError.UPLOAD_CANCELLED
)
) {
log.error("putFile to dataStore failed ", e);
}
throw e;
@@ -155,17 +162,19 @@ export class PhotosUploadHttpClient {
async putFilePart(
partUploadURL: string,
filePart: Uint8Array,
progressTracker,
progressTracker: unknown,
) {
try {
const response = await retryAsyncOperation(async () => {
const resp = await HTTPService.put(
partUploadURL,
filePart,
// @ts-ignore
null,
null,
progressTracker,
);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (!resp?.headers?.etag) {
const err = Error(CustomError.ETAG_MISSING);
log.error("putFile in parts failed", err);
@@ -175,7 +184,12 @@ export class PhotosUploadHttpClient {
}, handleUploadError);
return response.headers.etag as string;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
if (
!(
e instanceof Error &&
e.message == CustomError.UPLOAD_CANCELLED
)
) {
log.error("put filePart failed", e);
}
throw e;
@@ -185,7 +199,7 @@ export class PhotosUploadHttpClient {
async putFilePartV2(
partUploadURL: string,
filePart: Uint8Array,
progressTracker,
progressTracker: unknown,
) {
try {
const origin = await uploaderOrigin();
@@ -193,10 +207,12 @@ export class PhotosUploadHttpClient {
const resp = await HTTPService.put(
`${origin}/multipart-upload`,
filePart,
// @ts-ignore
null,
{ "UPLOAD-URL": partUploadURL },
progressTracker,
);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (!resp?.data?.etag) {
const err = Error(CustomError.ETAG_MISSING);
log.error("putFile in parts failed", err);
@@ -206,16 +222,22 @@ export class PhotosUploadHttpClient {
});
return response.data.etag as string;
} catch (e) {
if (e.message !== CustomError.UPLOAD_CANCELLED) {
if (
!(
e instanceof Error &&
e.message == CustomError.UPLOAD_CANCELLED
)
) {
log.error("put filePart failed", e);
}
throw e;
}
}
async completeMultipartUpload(completeURL: string, reqBody: any) {
async completeMultipartUpload(completeURL: string, reqBody: unknown) {
try {
await retryAsyncOperation(() =>
// @ts-ignore
HTTPService.post(completeURL, reqBody, null, {
"content-type": "text/xml",
}),
@@ -226,13 +248,14 @@ export class PhotosUploadHttpClient {
}
}
async completeMultipartUploadV2(completeURL: string, reqBody: any) {
async completeMultipartUploadV2(completeURL: string, reqBody: unknown) {
try {
const origin = await uploaderOrigin();
await retryAsyncOperation(() =>
HTTPService.post(
`${origin}/multipart-complete`,
reqBody,
// @ts-ignore
null,
{ "content-type": "text/xml", "UPLOAD-URL": completeURL },
),
@@ -244,25 +267,26 @@ export class PhotosUploadHttpClient {
}
}
export class PublicUploadHttpClient {
/**
* Lowest layer for file upload related HTTP operations when we're running in
* the context of the public albums app.
*/
export class PublicAlbumsUploadHTTPClient {
async uploadFile(
uploadFile: UploadFile,
token: string,
passwordToken: string,
credentials: PublicAlbumsCredentials,
): Promise<EnteFile> {
try {
if (!token) {
throw Error(CustomError.TOKEN_MISSING);
}
const url = await apiURL("/public-collection/file");
const response = await retryAsyncOperation(
() =>
HTTPService.post(url, uploadFile, null, {
"X-Auth-Access-Token": token,
...(passwordToken && {
"X-Auth-Access-Token-JWT": passwordToken,
}),
}),
HTTPService.post(
url,
uploadFile,
// @ts-ignore
null,
authenticatedPublicAlbumsRequestHeaders(credentials),
),
handleUploadError,
);
return response.data;
@@ -286,34 +310,19 @@ export class PublicUploadHttpClient {
headers: authenticatedPublicAlbumsRequestHeaders(credentials),
});
ensureOk(res);
return (
// TODO: The as cast will not be needed when tsc strict mode is
// enabled for this code.
ObjectUploadURLResponse.parse(await res.json())
.urls as ObjectUploadURL[]
);
return ObjectUploadURLResponse.parse(await res.json()).urls;
}
async fetchMultipartUploadURLs(
count: number,
token: string,
passwordToken: string,
credentials: PublicAlbumsCredentials,
): Promise<MultipartUploadURLs> {
try {
if (!token) {
throw Error(CustomError.TOKEN_MISSING);
}
const response = await HTTPService.get(
await apiURL("/public-collection/multipart-upload-urls"),
{ count },
{
"X-Auth-Access-Token": token,
...(passwordToken && {
"X-Auth-Access-Token-JWT": passwordToken,
}),
},
authenticatedPublicAlbumsRequestHeaders(credentials),
);
return response.data.urls;
} catch (e) {
log.error("fetch public multipart-upload-url failed", e);

40
web/packages/gallery/services/upload/takeout.ts
View File

@@ -1,8 +1,3 @@
// TODO: Audit this file
/* eslint-disable @typescript-eslint/ban-ts-comment */
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/dot-notation */
/** @file Dealing with the JSON metadata sidecar files */
import { ensureElectron } from "ente-base/electron";
@@ -157,33 +152,36 @@ const uploadItemText = async (uploadItem: UploadItem) => {
};
const parseMetadataJSONText = (text: string) => {
const metadataJSON: object = JSON.parse(text);
if (!metadataJSON) {
return undefined;
}
const metadataJSON_ = JSON.parse(text) as unknown;
// Ignore non objects.
if (typeof metadataJSON_ != "object") return undefined;
// Ignore null.
if (!metadataJSON_) return undefined;
// Ignore arrays.
if (Array.isArray(metadataJSON_)) return undefined;
// At this point, `metadataJSON_` is an `object`, but TypeScript won't let
// me index it. The following is the simplest (but unsafe) way I could think
// of for convincing TypeScript to allow me to index `metadataJSON` with
// `string`s.
const metadataJSON = metadataJSON_ as Record<string, unknown>;
const parsedMetadataJSON: ParsedMetadataJSON = {};
parsedMetadataJSON.creationTime =
// @ts-ignore
parseGTTimestamp(metadataJSON["photoTakenTime"]) ??
// @ts-ignore
parseGTTimestamp(metadataJSON["creationTime"]);
parseGTTimestamp(metadataJSON.photoTakenTime) ??
parseGTTimestamp(metadataJSON.creationTime);
parsedMetadataJSON.modificationTime = parseGTTimestamp(
// @ts-ignore
metadataJSON["modificationTime"],
metadataJSON.modificationTime,
);
parsedMetadataJSON.location =
// @ts-ignore
parseGTLocation(metadataJSON["geoData"]) ??
// @ts-ignore
parseGTLocation(metadataJSON["geoDataExif"]);
parseGTLocation(metadataJSON.geoData) ??
parseGTLocation(metadataJSON.geoDataExif);
parsedMetadataJSON.description = parseGTNonEmptyString(
// @ts-ignore
metadataJSON["description"],
metadataJSON.description,
);
return parsedMetadataJSON;

4
web/packages/gallery/services/upload/thumbnail.ts
View File

@@ -1,5 +1,3 @@
// TODO: Audit this file
/* eslint-disable @typescript-eslint/prefer-promise-reject-errors */
import log from "ente-base/log";
import { type Electron } from "ente-base/types/ipc";
import * as ffmpeg from "ente-gallery/services/ffmpeg";
@@ -92,6 +90,7 @@ const generateImageThumbnailUsingCanvas = async (blob: Blob) => {
canvasCtx.drawImage(image, 0, 0, width, height);
resolve(undefined);
} catch (e: unknown) {
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
reject(e);
}
};
@@ -163,6 +162,7 @@ export const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
canvasCtx.drawImage(video, 0, 0, width, height);
resolve(undefined);
} catch (e) {
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
reject(e);
}
});

155
web/packages/gallery/services/upload/upload-service.ts
View File

@@ -1,11 +1,6 @@
/* eslint-disable @typescript-eslint/no-unnecessary-type-assertion */
/* eslint-disable @typescript-eslint/require-await */
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
// TODO: Audit this file
/* eslint-disable @typescript-eslint/no-base-to-string */
/* eslint-disable @typescript-eslint/ban-ts-comment */
/* eslint-disable @typescript-eslint/no-inferrable-types */
import { streamEncryptionChunkSize } from "ente-base/crypto/libsodium";
import type { BytesOrB64 } from "ente-base/crypto/types";
import { type CryptoWorker } from "ente-base/crypto/worker";
@@ -19,12 +14,6 @@ import {
getNonEmptyMagicMetadataProps,
updateMagicMetadata,
} from "ente-gallery/services/magic-metadata";
import type { UploadItem } from "ente-gallery/services/upload";
import {
RANDOM_PERCENTAGE_PROGRESS_FOR_PUT,
type LivePhotoAssets,
type UploadResult,
} from "ente-gallery/services/upload";
import {
detectFileTypeInfoFromChunk,
isFileTypeNotSupportedError,
@@ -52,11 +41,16 @@ import { CustomError, handleUploadError } from "ente-shared/error";
import { mergeUint8Arrays } from "ente-utils/array";
import { ensureInteger, ensureNumber } from "ente-utils/ensure";
import * as convert from "xml-js";
import type { UploadableUploadItem } from "../upload";
import type { UploadableUploadItem, UploadItem } from ".";
import {
RANDOM_PERCENTAGE_PROGRESS_FOR_PUT,
type LivePhotoAssets,
type UploadResult,
} from ".";
import { tryParseEpochMicrosecondsFromFileName } from "./date";
import {
PhotosUploadHttpClient,
PublicUploadHttpClient,
PhotosUploadHTTPClient,
PublicAlbumsUploadHTTPClient,
type ObjectUploadURL,
} from "./remote";
import type { ParsedMetadataJSON } from "./takeout";
@@ -67,8 +61,8 @@ import {
generateThumbnailWeb,
} from "./thumbnail";
const publicUploadHttpClient = new PublicUploadHttpClient();
const UploadHttpClient = new PhotosUploadHttpClient();
const photosHTTPClient = new PhotosUploadHTTPClient();
const publicAlbumsHTTPClient = new PublicAlbumsUploadHTTPClient();
/**
* A readable stream for a file, and its associated size and last modified time.
@@ -120,7 +114,7 @@ const multipartChunksPerPart = 5;
/** Upload files to cloud storage */
class UploadService {
private uploadURLs: ObjectUploadURL[] = [];
private pendingUploadCount: number = 0;
private pendingUploadCount = 0;
private publicAlbumsCredentials: PublicAlbumsCredentials | undefined;
private activeUploadURLRefill: Promise<void> | undefined;
@@ -149,7 +143,9 @@ class UploadService {
await this.refillUploadURLs();
this.ensureUniqueUploadURLs();
}
return this.uploadURLs.pop();
const url = this.uploadURLs.pop();
if (!url) throw new Error("Failed to obtain upload URL");
return url;
}
private async preFetchUploadURLs() {
@@ -164,17 +160,12 @@ class UploadService {
}
async uploadFile(uploadFile: UploadFile) {
if (this.publicAlbumsCredentials) {
return publicUploadHttpClient.uploadFile(
uploadFile,
// TODO: publicAlbumsCredentials
this.publicAlbumsCredentials.accessToken,
// @ts-ignore
this.publicAlbumsCredentials.accessTokenJWT,
);
} else {
return UploadHttpClient.uploadFile(uploadFile);
}
return this.publicAlbumsCredentials
? publicAlbumsHTTPClient.uploadFile(
uploadFile,
this.publicAlbumsCredentials,
)
: photosHTTPClient.uploadFile(uploadFile);
}
private async refillUploadURLs() {
@@ -189,8 +180,8 @@ class UploadService {
}
private ensureUniqueUploadURLs() {
// TODO: Sanity check added on new implementation Nov 2024, remove after
// a while (tag: Migration).
// Sanity check added when this was a new implementation. Have kept it
// around, but it can be removed too.
if (
this.uploadURLs.length !=
new Set(this.uploadURLs.map((u) => u.url)).size
@@ -202,12 +193,12 @@ class UploadService {
private async _refillUploadURLs() {
let urls: ObjectUploadURL[];
if (this.publicAlbumsCredentials) {
urls = await publicUploadHttpClient.fetchUploadURLs(
urls = await publicAlbumsHTTPClient.fetchUploadURLs(
this.pendingUploadCount,
this.publicAlbumsCredentials,
);
} else {
urls = await UploadHttpClient.fetchUploadURLs(
urls = await photosHTTPClient.fetchUploadURLs(
this.pendingUploadCount,
);
}
@@ -215,17 +206,12 @@ class UploadService {
}
async fetchMultipartUploadURLs(count: number) {
if (this.publicAlbumsCredentials) {
// TODO: publicAlbumsCredentials
return await publicUploadHttpClient.fetchMultipartUploadURLs(
count,
this.publicAlbumsCredentials.accessToken,
// @ts-ignore
this.publicAlbumsCredentials.accessTokenJWT,
);
} else {
return await UploadHttpClient.fetchMultipartUploadURLs(count);
}
return this.publicAlbumsCredentials
? publicAlbumsHTTPClient.fetchMultipartUploadURLs(
count,
this.publicAlbumsCredentials,
)
: photosHTTPClient.fetchMultipartUploadURLs(count);
}
}
@@ -514,7 +500,7 @@ const uploadItemCreationDate = async (
parsedMetadataJSON: ParsedMetadataJSON | undefined,
) => {
if (parsedMetadataJSON?.creationTime)
return parsedMetadataJSON?.creationTime;
return parsedMetadataJSON.creationTime;
let parsedMetadata: ParsedMetadata | undefined;
if (fileType == FileType.image) {
@@ -522,7 +508,9 @@ const uploadItemCreationDate = async (
} else if (fileType == FileType.video) {
parsedMetadata = await tryExtractVideoMetadata(uploadItem);
} else {
throw new Error(`Unexpected file type ${fileType} for ${uploadItem}`);
throw new Error(
`Unexpected file type ${fileType} for ${uploadItemFileName(uploadItem)}`,
);
}
return parsedMetadata?.creationDate?.timestamp;
@@ -553,7 +541,7 @@ interface UploadResponse {
* {@link UploadManager} after it has assembled all the relevant bits we need to
* go forth and upload.
*/
export const uploader = async (
export const upload = async (
{ collection, localID, fileName, ...uploadAsset }: UploadableUploadItem,
uploaderName: string,
existingFiles: EnteFile[],
@@ -613,7 +601,7 @@ export const uploader = async (
areFilesSame(file.metadata, metadata),
);
const anyMatch = matches?.length > 0 ? matches[0] : undefined;
const anyMatch = matches.length > 0 ? matches[0] : undefined;
if (anyMatch) {
const matchInSameCollection = matches.find(
@@ -681,11 +669,10 @@ export const uploader = async (
uploadResult: metadata.hasStaticThumbnail
? "uploadedWithStaticThumbnail"
: "uploaded",
uploadedFile: uploadedFile,
uploadedFile,
};
} catch (e) {
// @ts-ignore
if (e.message == CustomError.UPLOAD_CANCELLED) {
if (e instanceof Error && e.message == CustomError.UPLOAD_CANCELLED) {
log.info(`Upload for ${fileName} cancelled`);
} else {
log.error(`Upload failed for ${fileName}`, e);
@@ -785,8 +772,7 @@ const readUploadItem = async (uploadItem: UploadItem): Promise<FileStream> => {
size,
lastModifiedMs: lm,
} = await readStream(ensureElectron(), uploadItem);
// @ts-ignore
underlyingStream = response.body;
underlyingStream = response.body!;
fileSize = size;
lastModifiedMs = lm;
} else {
@@ -808,7 +794,7 @@ const readUploadItem = async (uploadItem: UploadItem): Promise<FileStream> => {
// smaller).
let pending: Uint8Array | undefined;
const transformer = new TransformStream<Uint8Array, Uint8Array>({
async transform(
transform(
chunk: Uint8Array,
controller: TransformStreamDefaultController,
) {
@@ -891,8 +877,7 @@ const readImageOrVideoDetails = async (uploadItem: UploadItem) => {
// @ts-ignore
const fileTypeInfo = await detectFileTypeInfoFromChunk(async () => {
const reader = stream.getReader();
// @ts-ignore
const chunk = (await reader.read())!.value;
const chunk = (await reader.read()).value;
await reader.cancel();
return chunk;
}, uploadItemFileName(uploadItem));
@@ -1015,7 +1000,9 @@ const extractImageOrVideoMetadata = async (
} else if (fileType == FileType.video) {
parsedMetadata = await tryExtractVideoMetadata(uploadItem);
} else {
throw new Error(`Unexpected file type ${fileType} for ${uploadItem}`);
throw new Error(
`Unexpected file type ${fileType} for ${uploadItemFileName(uploadItem)}`,
);
}
// The `UploadAsset` itself might have metadata associated with a-priori, if
@@ -1049,9 +1036,7 @@ const extractImageOrVideoMetadata = async (
creationTime = timestamp;
publicMagicMetadata.dateTime = dateTime;
if (offset) publicMagicMetadata.offsetTime = offset;
// @ts-ignore
} else if (parsedMetadata.creationTime) {
// @ts-ignore
} else if (parsedMetadata?.creationTime) {
creationTime = parsedMetadata.creationTime;
} else {
creationTime =
@@ -1099,7 +1084,7 @@ const extractImageOrVideoMetadata = async (
const tryExtractImageMetadata = async (
uploadItem: UploadItem,
lastModifiedMs: number | undefined,
): Promise<ParsedMetadata> => {
): Promise<ParsedMetadata | undefined> => {
let file: File;
if (typeof uploadItem == "string" || Array.isArray(uploadItem)) {
// The library we use for extracting Exif from images, ExifReader,
@@ -1118,8 +1103,8 @@ const tryExtractImageMetadata = async (
try {
return await extractExif(file);
} catch (e) {
log.error(`Failed to extract image metadata for ${uploadItem}`, e);
// @ts-ignore
const fileName = uploadItemFileName(uploadItem);
log.error(`Failed to extract image metadata for ${fileName}`, e);
return undefined;
}
};
@@ -1128,7 +1113,8 @@ const tryExtractVideoMetadata = async (uploadItem: UploadItem) => {
try {
return await extractVideoMetadata(uploadItem);
} catch (e) {
log.error(`Failed to extract video metadata for ${uploadItem}`, e);
const fileName = uploadItemFileName(uploadItem);
log.error(`Failed to extract video metadata for ${fileName}`, e);
return undefined;
}
};
@@ -1184,6 +1170,7 @@ const readLivePhoto = async (
hasStaticThumbnail,
} = await withThumbnail(
livePhotoAssets.image,
// TODO: Update underlying type
// @ts-ignore
{ extension: fileTypeInfo.imageType, fileType: FileType.image },
await readUploadItem(livePhotoAssets.image),
@@ -1287,8 +1274,9 @@ const withThumbnail = async (
// directly for subsequent steps.
fileData = data;
} else {
const fileName = uploadItemFileName(uploadItem);
log.warn(
`Not using browser based thumbnail generation fallback for video at path ${uploadItem}`,
`Not using browser based thumbnail generation fallback for video at path ${fileName}`,
);
}
}
@@ -1320,7 +1308,7 @@ const constructPublicMagicMetadata = async (
publicMagicMetadataProps,
);
if (Object.values(nonEmptyPublicMagicMetadataProps)?.length === 0) {
if (Object.values(nonEmptyPublicMagicMetadataProps).length === 0) {
// @ts-ignore
return null;
}
@@ -1353,6 +1341,8 @@ const encryptFile = async (
});
let encryptedPubMagicMetadata: EncryptedMagicMetadata;
// Keep defensive check until the underlying type is audited.
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (pubMagicMetadata) {
const encryptedPubMagicMetadataData = await worker.encryptMetadataJSON({
jsonValue: pubMagicMetadata.data,
@@ -1424,8 +1414,7 @@ const uploadToBucket = async (
const { localID, file, thumbnail, metadata, pubMagicMetadata } =
encryptedFilePieces;
try {
// @ts-ignore
let fileObjectKey: string = null;
let fileObjectKey: string;
let fileSize: number;
const encryptedData = file.encryptedData;
@@ -1453,15 +1442,13 @@ const uploadToBucket = async (
const progressTracker = makeProgressTracker(localID);
const fileUploadURL = await uploadService.getUploadURL();
if (!isCFUploadProxyDisabled) {
fileObjectKey = await UploadHttpClient.putFileV2(
// @ts-ignore
fileObjectKey = await photosHTTPClient.putFileV2(
fileUploadURL,
data,
progressTracker,
);
} else {
fileObjectKey = await UploadHttpClient.putFile(
// @ts-ignore
fileObjectKey = await photosHTTPClient.putFile(
fileUploadURL,
data,
progressTracker,
@@ -1469,18 +1456,15 @@ const uploadToBucket = async (
}
}
const thumbnailUploadURL = await uploadService.getUploadURL();
// @ts-ignore
let thumbnailObjectKey: string = null;
let thumbnailObjectKey: string;
if (!isCFUploadProxyDisabled) {
thumbnailObjectKey = await UploadHttpClient.putFileV2(
// @ts-ignore
thumbnailObjectKey = await photosHTTPClient.putFileV2(
thumbnailUploadURL,
thumbnail.encryptedData,
null,
);
} else {
thumbnailObjectKey = await UploadHttpClient.putFile(
// @ts-ignore
thumbnailObjectKey = await photosHTTPClient.putFile(
thumbnailUploadURL,
thumbnail.encryptedData,
null,
@@ -1506,8 +1490,9 @@ const uploadToBucket = async (
};
return backupedFile;
} catch (e) {
// @ts-ignore
if (e.message !== CustomError.UPLOAD_CANCELLED) {
if (
!(e instanceof Error && e.message == CustomError.UPLOAD_CANCELLED)
) {
log.error("Error when uploading to bucket", e);
}
throw e;
@@ -1554,13 +1539,13 @@ async function uploadStreamUsingMultipart(
);
let eTag = null;
if (!isCFUploadProxyDisabled) {
eTag = await UploadHttpClient.putFilePartV2(
eTag = await photosHTTPClient.putFilePartV2(
fileUploadURL,
uploadChunk,
progressTracker,
);
} else {
eTag = await UploadHttpClient.putFilePart(
eTag = await photosHTTPClient.putFilePart(
fileUploadURL,
uploadChunk,
progressTracker,
@@ -1577,9 +1562,9 @@ async function uploadStreamUsingMultipart(
{ compact: true, ignoreComment: true, spaces: 4 },
);
if (!isCFUploadProxyDisabled) {
await UploadHttpClient.completeMultipartUploadV2(completeURL, cBody);
await photosHTTPClient.completeMultipartUploadV2(completeURL, cBody);
} else {
await UploadHttpClient.completeMultipartUpload(completeURL, cBody);
await photosHTTPClient.completeMultipartUpload(completeURL, cBody);
}
return { objectKey: multipartUploadURLs.objectKey, fileSize };