TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[web] File internals cleanup - Part 3/x (#6345)

Browse Source
This commit is contained in:
Manav Rathi
2025-06-23 18:10:11 +05:30
committed by GitHub
parent 493fe4b3a5 fa137dcccc
commit e603853e52
22 changed files with 322 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
web/apps/photos/src/components/FileList.tsx
View File

@@ -26,7 +26,7 @@ import {
} from "ente-new/photos/components/PlaceholderThumbnails";
import { TileBottomTextOverlay } from "ente-new/photos/components/Tiles";
import { PseudoCollectionID } from "ente-new/photos/services/collection-summary";
import { enteFileDeletionDate } from "ente-new/photos/services/files";
import { type EnteTrashFile } from "ente-new/photos/services/trash";
import { t } from "i18next";
import memoize from "memoize-one";
import { GalleryContext } from "pages/gallery";
@@ -355,21 +355,19 @@ export const FileList: React.FC<FileListProps> = ({
const groupByTime = (timeStampList: TimeStampListItem[]) => {
let listItemIndex = 0;
let currentDate;
let lastCreationTime: number | undefined;
annotatedFiles.forEach((item, index) => {
const creationTime = fileCreationTime(item.file) / 1000;
if (
!currentDate ||
!isSameDay(
new Date(fileCreationTime(item.file) / 1000),
new Date(currentDate),
)
!lastCreationTime ||
!isSameDay(new Date(creationTime), new Date(lastCreationTime))
) {
currentDate = fileCreationTime(item.file) / 1000;
lastCreationTime = creationTime;
timeStampList.push({
tag: "date",
date: item.timelineDateString,
id: currentDate.toString(),
id: lastCreationTime.toString(),
});
timeStampList.push({
tag: "file",
@@ -1285,12 +1283,12 @@ const FileThumbnail: React.FC<FileThumbnailProps> = ({
{activeCollectionID == PseudoCollectionID.trash &&
// TODO(RE):
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
file.isTrashed && (
(file as EnteTrashFile).deleteBy && (
<TileBottomTextOverlay>
<Typography variant="small">
{formattedDateRelative(enteFileDeletionDate(file))}
{formattedDateRelative(
(file as EnteTrashFile).deleteBy,
)}
</Typography>
</TileBottomTextOverlay>
)}

24
web/apps/photos/src/components/UploadProgress.tsx
View File

@@ -45,7 +45,7 @@ import {
ListItemKeySelector,
} from "react-window";
import type {
FinishedUploadResult,
FinishedUploadType,
InProgressUpload,
SegregatedFinishedUploads,
UploadCounter,
@@ -321,11 +321,11 @@ function UploadProgressDialog() {
<DialogContent sx={{ "&&&": { px: 0 } }}>
{uploadPhase == "uploading" && <InProgressSection />}
<ResultSection
uploadResult="uploaded"
resultType="uploaded"
sectionTitle={t("successful_uploads")}
/>
<ResultSection
uploadResult="uploadedWithStaticThumbnail"
resultType="uploadedWithStaticThumbnail"
sectionTitle={t("thumbnail_generation_failed")}
sectionInfo={t("thumbnail_generation_failed_hint")}
/>
@@ -335,12 +335,12 @@ function UploadProgressDialog() {
</NotUploadSectionHeader>
)}
<ResultSection
uploadResult="blocked"
resultType="blocked"
sectionTitle={t("blocked_uploads")}
sectionInfo={<Trans i18nKey={"blocked_uploads_hint"} />}
/>
<ResultSection
uploadResult="failed"
resultType="failed"
sectionTitle={t("failed_uploads")}
sectionInfo={
uploadPhase == "done"
@@ -349,22 +349,22 @@ function UploadProgressDialog() {
}
/>
<ResultSection
uploadResult="alreadyUploaded"
resultType="alreadyUploaded"
sectionTitle={t("ignored_uploads")}
sectionInfo={t("ignored_uploads_hint")}
/>
<ResultSection
uploadResult="largerThanAvailableStorage"
resultType="largerThanAvailableStorage"
sectionTitle={t("insufficient_storage")}
sectionInfo={t("insufficient_storage_hint")}
/>
<ResultSection
uploadResult="unsupported"
resultType="unsupported"
sectionTitle={t("unsupported_files")}
sectionInfo={t("unsupported_files_hint")}
/>
<ResultSection
uploadResult="tooLarge"
resultType="tooLarge"
sectionTitle={t("large_files")}
sectionInfo={t("large_files_hint")}
/>
@@ -482,20 +482,20 @@ const NotUploadSectionHeader = styled("div")(
);
interface ResultSectionProps {
uploadResult: FinishedUploadResult;
resultType: FinishedUploadType;
sectionTitle: string;
sectionInfo?: React.ReactNode;
}
const ResultSection: React.FC<ResultSectionProps> = ({
uploadResult,
resultType,
sectionTitle,
sectionInfo,
}) => {
const { finishedUploads, uploadFileNames } = useContext(
UploadProgressContext,
);
const fileList = finishedUploads.get(uploadResult);
const fileList = finishedUploads.get(resultType);
if (!fileList?.length) {
return <></>;

2
web/apps/photos/src/pages/gallery.tsx
View File

@@ -75,7 +75,6 @@ import {
} from "ente-new/photos/services/collection-summary";
import exportService from "ente-new/photos/services/export";
import { updateFilesVisibility } from "ente-new/photos/services/file";
import { getLocalTrashedFiles } from "ente-new/photos/services/files";
import {
savedCollections,
savedHiddenFiles,
@@ -92,6 +91,7 @@ import {
preCollectionAndFilesSync,
syncCollectionAndFiles,
} from "ente-new/photos/services/sync";
import { getLocalTrashedFiles } from "ente-new/photos/services/trash";
import {
initUserDetailsOrTriggerSync,
redirectToCustomerPortal,

2
web/apps/photos/src/pages/shared-albums.tsx
View File

@@ -47,6 +47,7 @@ import { FullScreenDropZone } from "ente-gallery/components/FullScreenDropZone";
import { downloadManager } from "ente-gallery/services/download";
import { extractCollectionKeyFromShareURL } from "ente-gallery/services/share";
import { updateShouldDisableCFUploadProxy } from "ente-gallery/services/upload";
import { sortFiles } from "ente-gallery/utils/files";
import type { Collection } from "ente-media/collection";
import { type EnteFile } from "ente-media/file";
import { verifyPublicAlbumPassword } from "ente-new/albums/services/publicCollection";
@@ -56,7 +57,6 @@ import {
} from "ente-new/photos/components/gallery/ListHeader";
import { isHiddenCollection } from "ente-new/photos/services/collection";
import { PseudoCollectionID } from "ente-new/photos/services/collection-summary";
import { sortFiles } from "ente-new/photos/services/files";
import { usePhotosAppContext } from "ente-new/photos/types/context";
import { CustomError, parseSharingErrorCodes } from "ente-shared/error";
import { t } from "i18next";

7
web/apps/photos/src/services/collectionService.ts
View File

@@ -2,6 +2,7 @@ import type { User } from "ente-accounts/services/user";
import { ensureLocalUser } from "ente-accounts/services/user";
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { groupFilesByCollectionID, sortFiles } from "ente-gallery/utils/files";
import { Collection } from "ente-media/collection";
import { EnteFile } from "ente-media/file";
import {
@@ -18,11 +19,7 @@ import {
CollectionsSortBy,
} from "ente-new/photos/services/collection-summary";
import { getLocalCollections } from "ente-new/photos/services/collections";
import {
getLocalFiles,
groupFilesByCollectionID,
sortFiles,
} from "ente-new/photos/services/files";
import { getLocalFiles } from "ente-new/photos/services/files";
import HTTPService from "ente-shared/network/HTTPService";
import { getData } from "ente-shared/storage/localStorage";
import { getToken } from "ente-shared/storage/localStorage/helpers";

2
web/apps/photos/src/services/publicCollectionService.ts
View File

@@ -3,6 +3,7 @@ import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { transformFilesIfNeeded } from "ente-gallery/services/files-db";
import { type MagicMetadataCore } from "ente-gallery/services/magic-metadata";
import { sortFiles } from "ente-gallery/utils/files";
import type {
Collection,
CollectionPublicMagicMetadataData,
@@ -10,7 +11,6 @@ import type {
import type { EnteFile, RemoteEnteFile } from "ente-media/file";
import { decryptRemoteFile } from "ente-media/file";
import { savedPublicCollections } from "ente-new/albums/services/public-albums-fdb";
import { sortFiles } from "ente-new/photos/services/files";
import { CustomError, parseSharingErrorCodes } from "ente-shared/error";
import HTTPService from "ente-shared/network/HTTPService";
import localForage from "ente-shared/storage/localForage";

113
web/apps/photos/src/services/upload-manager.ts
View File

@@ -29,11 +29,7 @@ import UploadService, {
} from "ente-gallery/services/upload/upload-service";
import { processVideoNewUpload } from "ente-gallery/services/video";
import type { Collection } from "ente-media/collection";
import {
decryptRemoteFile,
type EnteFile,
type RemoteEnteFile,
} from "ente-media/file";
import { type EnteFile } from "ente-media/file";
import {
fileCreationTime,
type ParsedMetadata,
@@ -67,22 +63,17 @@ export interface InProgressUpload {
}
/**
* A variant of {@link UploadResult} used when segregating finished uploads in
* the UI. "addedSymlink" is treated as "uploaded", everything else remains as
* it were.
* A variant of {@link UploadResult}'s {@link type} values used when segregating
* finished uploads in the UI. "addedSymlink" is treated as "uploaded",
* everything else remains as it were.
*/
export type FinishedUploadResult = Exclude<UploadResult, "addedSymlink">;
export interface FinishedUpload {
localFileID: FileID;
result: FinishedUploadResult;
}
export type FinishedUploadType = Exclude<UploadResult["type"], "addedSymlink">;
export type InProgressUploads = Map<FileID, PercentageUploaded>;
export type FinishedUploads = Map<FileID, FinishedUploadResult>;
export type FinishedUploads = Map<FileID, FinishedUploadType>;
export type SegregatedFinishedUploads = Map<FinishedUploadResult, FileID[]>;
export type SegregatedFinishedUploads = Map<FinishedUploadType, FileID[]>;
export interface ProgressUpdater {
setPercentComplete: React.Dispatch<React.SetStateAction<number>>;
@@ -145,7 +136,7 @@ class UIService {
this.setTotalFileCount(count);
this.filesUploadedCount = 0;
this.inProgressUploads = new Map<number, number>();
this.finishedUploads = new Map<number, FinishedUploadResult>();
this.finishedUploads = new Map<number, FinishedUploadType>();
this.updateProgressBarUI();
}
@@ -189,8 +180,8 @@ class UIService {
this.updateProgressBarUI();
}
moveFileToResultList(key: number, uploadResult: FinishedUploadResult) {
this.finishedUploads.set(key, uploadResult);
moveFileToResultList(key: number, type: FinishedUploadType) {
this.finishedUploads.set(key, type);
this.inProgressUploads.delete(key);
this.updateProgressBarUI();
}
@@ -523,7 +514,7 @@ class UploadManager {
uiService.setFileProgress(localID, 0);
await wait(0);
const { uploadResult, uploadedFile } = await upload(
const uploadResult = await upload(
uploadableItem,
this.uploaderName,
this.existingFiles,
@@ -532,13 +523,12 @@ class UploadManager {
uploadContext,
);
const finalUploadResult = await this.postUploadTask(
const finishedUploadType = await this.postUploadTask(
uploadableItem,
uploadResult,
uploadedFile,
);
uiService.moveFileToResultList(localID, finalUploadResult);
uiService.moveFileToResultList(localID, finishedUploadType);
uiService.increaseFileUploaded();
UploadService.reducePendingUploadCount();
}
@@ -547,72 +537,42 @@ class UploadManager {
private async postUploadTask(
uploadableItem: UploadableUploadItem,
uploadResult: UploadResult,
uploadedFile: RemoteEnteFile | EnteFile | undefined,
): Promise<FinishedUploadResult> {
log.info(`Upload ${uploadableItem.fileName} | ${uploadResult}`);
const finishedUploadResult =
uploadResult == "addedSymlink" ? "uploaded" : uploadResult;
): Promise<FinishedUploadType> {
const type = uploadResult.type;
log.info(`Upload ${uploadableItem.fileName} | ${type}`);
try {
const processableUploadItem =
await markUploadedAndObtainProcessableItem(uploadableItem);
let decryptedFile: EnteFile;
switch (uploadResult) {
switch (uploadResult.type) {
case "failed":
case "blocked":
// Retriable error.
this.failedItems.push(uploadableItem);
break;
case "alreadyUploaded":
case "addedSymlink":
decryptedFile = uploadedFile as EnteFile;
this.updateExistingFiles(uploadResult.file);
break;
case "uploaded":
case "uploadedWithStaticThumbnail":
decryptedFile = await decryptRemoteFile(
uploadedFile as RemoteEnteFile,
uploadableItem.collection.key,
);
{
const { file } = uploadResult;
indexNewUpload(file, processableUploadItem);
processVideoNewUpload(file, processableUploadItem);
this.updateExistingFiles(file);
}
break;
case "largerThanAvailableStorage":
case "unsupported":
case "tooLarge":
// no-op
break;
default:
throw new Error(`Invalid Upload Result ${uploadResult}`);
}
if (
[
"addedSymlink",
"uploaded",
"uploadedWithStaticThumbnail",
].includes(uploadResult)
) {
const uploadItem =
uploadableItem.uploadItem ??
uploadableItem.livePhotoAssets.image;
if (
uploadItem &&
(uploadResult == "uploaded" ||
uploadResult == "uploadedWithStaticThumbnail")
) {
indexNewUpload(decryptedFile, processableUploadItem);
processVideoNewUpload(decryptedFile, processableUploadItem);
}
this.updateExistingFiles(decryptedFile);
}
if (isDesktop) {
if (watcher.isUploadRunning()) {
await watcher.onFileUpload(
uploadResult,
uploadableItem,
uploadedFile,
);
}
if (isDesktop && watcher.isUploadRunning()) {
await watcher.onFileUpload(uploadableItem, uploadResult);
}
return finishedUploadResult;
return type == "addedSymlink" ? "uploaded" : type;
} catch (e) {
log.error("Post file upload action failed", e);
return "failed";
@@ -641,12 +601,9 @@ class UploadManager {
return this.uploaderName;
}
private updateExistingFiles(decryptedFile: EnteFile) {
if (!decryptedFile) {
throw Error("decrypted file can't be undefined");
}
this.existingFiles.push(decryptedFile);
this.onUploadFile(decryptedFile);
private updateExistingFiles(file: EnteFile) {
this.existingFiles.push(file);
this.onUploadFile(file);
}
/**

95
web/apps/photos/src/services/watch.ts
View File

@@ -14,19 +14,13 @@ import type {
} from "ente-base/types/ipc";
import { type UploadResult } from "ente-gallery/services/upload";
import type { UploadAsset } from "ente-gallery/services/upload/upload-service";
import {
getLocalFiles,
groupFilesByCollectionID,
} from "ente-new/photos/services/files";
import { groupFilesByCollectionID } from "ente-gallery/utils/files";
import type { EnteFile } from "ente-media/file";
import { getLocalFiles } from "ente-new/photos/services/files";
import { ensureString } from "ente-utils/ensure";
import { removeFromCollection } from "./collectionService";
import { type UploadItemWithCollection, uploadManager } from "./upload-manager";
interface FolderWatchUploadedFile {
id: number;
collectionID: number;
}
/**
* Watch for file system folders and automatically update the corresponding Ente
* collections.
@@ -53,7 +47,7 @@ class FolderWatcher {
* A map from file paths to the (fileID, collectionID) of the file that was
* uploaded (or symlinked) as part of the most recent upload attempt.
*/
private uploadedFileForPath = new Map<string, FolderWatchUploadedFile>();
private uploadedFileForPath = new Map<string, EnteFile>();
/**
* A set of file paths that could not be uploaded in the most recent upload
* attempt. These are the uploads that failed due to a permanent error that
@@ -326,47 +320,54 @@ class FolderWatcher {
* {@link upload} gets uploaded.
*/
async onFileUpload(
fileUploadResult: UploadResult,
item: UploadItemWithCollection,
file: FolderWatchUploadedFile,
uploadResult: UploadResult,
) {
// Re the usage of ensureString: For desktop watch, the only possibility
// for a UploadItem is for it to be a string (the absolute path to a
// file on disk).
if (
[
"addedSymlink",
"uploaded",
"uploadedWithStaticThumbnail",
"alreadyUploaded",
].includes(fileUploadResult)
) {
if (item.isLivePhoto) {
this.uploadedFileForPath.set(
ensureString(item.livePhotoAssets.image),
file,
);
this.uploadedFileForPath.set(
ensureString(item.livePhotoAssets.video),
file,
);
} else {
this.uploadedFileForPath.set(
ensureString(item.uploadItem),
file,
);
}
} else if (["unsupported", "tooLarge"].includes(fileUploadResult)) {
if (item.isLivePhoto) {
this.unUploadableFilePaths.add(
ensureString(item.livePhotoAssets.image),
);
this.unUploadableFilePaths.add(
ensureString(item.livePhotoAssets.video),
);
} else {
this.unUploadableFilePaths.add(ensureString(item.uploadItem));
}
switch (uploadResult.type) {
case "alreadyUploaded":
case "addedSymlink":
case "uploaded":
case "uploadedWithStaticThumbnail":
{
// Done.
if (item.isLivePhoto) {
this.uploadedFileForPath.set(
ensureString(item.livePhotoAssets.image),
uploadResult.file,
);
this.uploadedFileForPath.set(
ensureString(item.livePhotoAssets.video),
uploadResult.file,
);
} else {
this.uploadedFileForPath.set(
ensureString(item.uploadItem),
uploadResult.file,
);
}
}
break;
case "unsupported":
case "tooLarge":
{
// Non-retriable error.
if (item.isLivePhoto) {
this.unUploadableFilePaths.add(
ensureString(item.livePhotoAssets.image),
);
this.unUploadableFilePaths.add(
ensureString(item.livePhotoAssets.video),
);
} else {
this.unUploadableFilePaths.add(
ensureString(item.uploadItem),
);
}
}
break;
}
}
@@ -408,7 +409,7 @@ class FolderWatcher {
const syncedFiles: FolderWatch["syncedFiles"] = [];
const ignoredFiles: FolderWatch["ignoredFiles"] = [];
const markSynced = (file: FolderWatchUploadedFile, path: string) => {
const markSynced = (file: EnteFile, path: string) => {
syncedFiles.push({
path,
uploadedFileID: file.id,

2
web/apps/photos/tests/upload.test.ts
View File

@@ -7,6 +7,7 @@ import {
matchJSONMetadata,
metadataJSONMapKeyForJSON,
} from "ente-gallery/services/upload/metadata-json";
import { groupFilesByCollectionID } from "ente-gallery/utils/files";
import {
fileCreationTime,
fileFileName,
@@ -14,7 +15,6 @@ import {
} from "ente-media/file-metadata";
import { FileType } from "ente-media/file-type";
import { getLocalCollections } from "ente-new/photos/services/collections";
import { groupFilesByCollectionID } from "ente-new/photos/services/files";
import { savedNormalFiles } from "ente-new/photos/services/photos-fdb";
import { getUserDetailsV2 } from "ente-new/photos/services/user-details";

17
web/packages/base/date.ts
View File

@@ -1,20 +1,3 @@
/**
* Convert an epoch microsecond value to a JavaScript date.
*
* [Note: Remote timestamps are epoch microseconds]
*
* This is a convenience API for dealing with optional epoch microseconds in
* various data structures. Remote talks in terms of epoch microseconds, but
* JavaScript dates are underlain by epoch milliseconds, and this does a
* conversion, with a convenience of short circuiting undefined values.
*/
export const dateFromEpochMicroseconds = (
epochMicroseconds: number | undefined,
) =>
epochMicroseconds === undefined
? undefined
: new Date(epochMicroseconds / 1000);
/**
* Return `true` if both the given dates have the same day.
*/

23
web/packages/base/i18n-date.ts
View File

@@ -57,11 +57,14 @@ export const formattedTime = (date: Date) => _timeFormat.format(date);
* @param dateOrEpochMicroseconds A JavaScript Date or a numeric epoch
* microseconds value.
*
* [Note: Remote timestamps are epoch microseconds]
*
* Remote talks in terms of epoch microseconds, while JavaScript dates are
* underlain by epoch milliseconds.
*
* As a convenience, this function can be either be directly passed a JavaScript
* date, or it can be given the raw epoch microseconds value and it'll convert
* internally.
*
* See: [Note: Remote timestamps are epoch microseconds]
*/
export const formattedDateTime = (dateOrEpochMicroseconds: Date | number) =>
_formattedDateTime(toDate(dateOrEpochMicroseconds));
@@ -74,7 +77,19 @@ const toDate = (dm: Date | number) =>
let _relativeTimeFormat: Intl.RelativeTimeFormat | undefined;
export const formattedDateRelative = (date: Date) => {
/**
* Return a locale aware relative version of the given date.
*
* Example: "in 23 days"
*
* @param dateOrEpochMicroseconds A JavaScript Date or a numeric epoch
* microseconds value.
*
* See: [Note: Remote timestamps are epoch microseconds]
*/
export const formattedDateRelative = (
dateOrEpochMicroseconds: Date | number,
) => {
const units: [Intl.RelativeTimeFormatUnit, number][] = [
["year", 24 * 60 * 60 * 1000 * 365],
["month", (24 * 60 * 60 * 1000 * 365) / 12],
@@ -84,6 +99,8 @@ export const formattedDateRelative = (date: Date) => {
["second", 1000],
];
const date = toDate(dateOrEpochMicroseconds);
// Math.abs accounts for both past and future scenarios.
const elapsed = Math.abs(date.getTime() - Date.now());

23
web/packages/gallery/services/upload/index.ts
View File

@@ -4,6 +4,7 @@ import { customAPIOrigin } from "ente-base/origins";
import type { ZipItem } from "ente-base/types/ipc";
import { exportMetadataDirectoryName } from "ente-gallery/export-dirs";
import type { Collection } from "ente-media/collection";
import type { EnteFile } from "ente-media/file";
import { nullToUndefined } from "ente-utils/transform";
import { z } from "zod/v4";
@@ -278,8 +279,8 @@ export type UploadableUploadItem = ClusteredUploadItem & {
};
/**
* Result of {@link markUploadedAndObtainPostProcessableItem}; see the
* documentation of that function for the meaning and cases of this type.
* Result of {@link markUploadedAndObtainProcessableItem}; see the documentation
* of that function for the meaning and cases of this type.
*/
export type ProcessableUploadItem = File | TimestampedFileSystemUploadItem;
@@ -432,15 +433,15 @@ export type UploadPhase =
| "done";
export type UploadResult =
| "failed"
| "alreadyUploaded"
| "unsupported"
| "blocked"
| "tooLarge"
| "largerThanAvailableStorage"
| "uploaded"
| "uploadedWithStaticThumbnail"
| "addedSymlink";
| { type: "unsupported" }
| { type: "tooLarge" }
| { type: "largerThanAvailableStorage" }
| { type: "blocked" }
| { type: "failed" }
| { type: "alreadyUploaded"; file: EnteFile }
| { type: "addedSymlink"; file: EnteFile }
| { type: "uploadedWithStaticThumbnail"; file: EnteFile }
| { type: "uploaded"; file: EnteFile };
/**
* Return true to disable the upload of files via Cloudflare Workers.

8
web/packages/gallery/services/upload/remote.ts
View File

@@ -7,7 +7,7 @@ import {
type PublicAlbumsCredentials,
} from "ente-base/http";
import { apiURL, uploaderOrigin } from "ente-base/origins";
import { type RemoteEnteFile, type RemoteFileMetadata } from "ente-media/file";
import { RemoteEnteFile, type RemoteFileMetadata } from "ente-media/file";
import type { RemoteMagicMetadata } from "ente-media/magic-metadata";
import { nullToUndefined } from "ente-utils/transform";
import { z } from "zod/v4";
@@ -480,8 +480,7 @@ export const postEnteFile = async (
body: JSON.stringify(postFileRequest),
});
ensureOk(res);
// TODO(RE):
return (await res.json()) as RemoteEnteFile;
return RemoteEnteFile.parse(await res.json());
};
/**
@@ -498,6 +497,5 @@ export const postPublicAlbumsEnteFile = async (
body: JSON.stringify(postFileRequest),
});
ensureOk(res);
// TODO(RE):
return (await res.json()) as RemoteEnteFile;
return RemoteEnteFile.parse(await res.json());
};

39
web/packages/gallery/services/upload/upload-service.ts
View File

@@ -24,7 +24,7 @@ import {
isFileTypeNotSupportedError,
} from "ente-gallery/utils/detect-type";
import { readStream } from "ente-gallery/utils/native-stream";
import type { EnteFile, RemoteEnteFile } from "ente-media/file";
import { decryptRemoteFile, type EnteFile } from "ente-media/file";
import {
fileFileName,
metadataHash,
@@ -42,8 +42,13 @@ import {
import { addToCollection } from "ente-new/photos/services/collection";
import { mergeUint8Arrays } from "ente-utils/array";
import { ensureInteger, ensureNumber } from "ente-utils/ensure";
import type { UploadableUploadItem, UploadItem, UploadPathPrefix } from ".";
import { type LivePhotoAssets, type UploadResult } from ".";
import type {
UploadableUploadItem,
UploadItem,
UploadPathPrefix,
UploadResult,
} from ".";
import { type LivePhotoAssets } from ".";
import { tryParseEpochMicrosecondsFromFileName } from "./date";
import { matchJSONMetadata, type ParsedMetadataJSON } from "./metadata-json";
import {
@@ -635,11 +640,6 @@ interface UploadContext {
updateUploadProgress: (fileLocalID: number, percentage: number) => void;
}
interface UploadResponse {
uploadResult: UploadResult;
uploadedFile?: RemoteEnteFile | EnteFile;
}
/**
* Upload the given {@link UploadableUploadItem}
*
@@ -657,7 +657,7 @@ export const upload = async (
parsedMetadataJSONMap: Map<string, ParsedMetadataJSON>,
worker: CryptoWorker,
uploadContext: UploadContext,
): Promise<UploadResponse> => {
): Promise<UploadResult> => {
const { abortIfCancelled } = uploadContext;
log.info(`Upload ${fileName} | start`);
@@ -685,7 +685,7 @@ export const upload = async (
} catch (e) {
if (isFileTypeNotSupportedError(e)) {
log.error(`Not uploading ${fileName}`, e);
return { uploadResult: "unsupported" };
return { type: "unsupported" };
}
throw e;
}
@@ -693,7 +693,7 @@ export const upload = async (
const { fileTypeInfo, fileSize, lastModifiedMs } = assetDetails;
const maxFileSize = 10 * 1024 * 1024 * 1024; /* 10 GB */
if (fileSize >= maxFileSize) return { uploadResult: "tooLarge" };
if (fileSize >= maxFileSize) return { type: "tooLarge" };
abortIfCancelled();
@@ -717,16 +717,13 @@ export const upload = async (
(f) => f.collectionID == collection.id,
);
if (matchInSameCollection) {
return {
uploadResult: "alreadyUploaded",
uploadedFile: matchInSameCollection,
};
return { type: "alreadyUploaded", file: matchInSameCollection };
} else {
// Any of the matching files can be used to add a symlink.
const symlink = Object.assign({}, anyMatch);
symlink.collectionID = collection.id;
await addToCollection(collection, [symlink]);
return { uploadResult: "addedSymlink", uploadedFile: symlink };
return { type: "addedSymlink", file: symlink };
}
}
@@ -778,10 +775,10 @@ export const upload = async (
);
return {
uploadResult: metadata.hasStaticThumbnail
type: metadata.hasStaticThumbnail
? "uploadedWithStaticThumbnail"
: "uploaded",
uploadedFile,
file: await decryptRemoteFile(uploadedFile, collection.key),
};
} catch (e) {
if (isUploadCancelledError(e)) {
@@ -799,11 +796,11 @@ export const upload = async (
/* file specific */
case eTagMissingErrorMessage:
return { uploadResult: "blocked" };
return { type: "blocked" };
case fileTooLargeErrorMessage:
return { uploadResult: "largerThanAvailableStorage" };
return { type: "largerThanAvailableStorage" };
default:
return { uploadResult: "failed" };
return { type: "failed" };
}
}
};

6
web/packages/gallery/services/video.ts
View File

@@ -9,14 +9,12 @@ import { getKV, getKVB, getKVN, setKV } from "ente-base/kv";
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import { ensureAuthToken } from "ente-base/token";
import { uniqueFilesByID } from "ente-gallery/utils/files";
import { fileLogID, type EnteFile } from "ente-media/file";
import { FileType } from "ente-media/file-type";
import { updateFilePublicMagicMetadata } from "ente-new/photos/services/file";
import {
getLocalTrashFileIDs,
uniqueFilesByID,
} from "ente-new/photos/services/files";
import { savedFiles } from "ente-new/photos/services/photos-fdb";
import { getLocalTrashFileIDs } from "ente-new/photos/services/trash";
import { gunzip, gzip } from "ente-new/photos/utils/gzip";
import { randomSample } from "ente-utils/array";
import { ensurePrecondition } from "ente-utils/ensure";

75
web/packages/gallery/utils/files.ts Normal file
View File

@@ -0,0 +1,75 @@
import type { EnteFile } from "ente-media/file";
import { fileCreationTime } from "ente-media/file-metadata";
/**
* Sort the given list of {@link EnteFile}s in place.
*
* Like the JavaScript Array#sort, this method modifies the {@link files}
* argument. It sorts {@link files} in place, and then returns a reference to
* the same mutated array.
*
* By default, files are sorted so that the newest one is first. The optional
* {@link sortAsc} flag can be set to `true` to sort them so that the oldest one
* is first.
*/
export const sortFiles = (files: EnteFile[], sortAsc = false) => {
// Sort based on the time of creation time of the file.
//
// For files with same creation time, sort based on the time of last
// modification.
const factor = sortAsc ? -1 : 1;
return files.sort((a, b) => {
const at = fileCreationTime(a);
const bt = fileCreationTime(b);
return at == bt
? factor *
(b.metadata.modificationTime - a.metadata.modificationTime)
: factor * (bt - at);
});
};
/**
* [Note: Collection file]
*
* File IDs themselves are unique across all the files for the user (in fact,
* they're unique across all the files in an Ente instance).
*
* However, we can have multiple entries for the same file ID in our local
* database and/or remote responses because the unit of account is not file, but
* a "Collection File" – a collection and file pair.
*
* For example, if the same file is symlinked into two collections, then we will
* have two "Collection File" entries for it, both with the same file ID, but
* with different collection IDs.
*
* This function returns files such that only one of these entries is returned.
* The entry that is returned is arbitrary in general, this function just picks
* the first one for each unique file ID.
*
* If this function is invoked on a list on which {@link sortFiles} has already
* been called, which by default sorts such that the newest file is first, then
* this function's behaviour would be to return the newest file from among
* multiple files with the same ID but different collections.
*/
export const uniqueFilesByID = (files: EnteFile[]) => {
const seen = new Set<number>();
return files.filter(({ id }) => {
if (seen.has(id)) return false;
seen.add(id);
return true;
});
};
/**
* Segment the given {@link files} into lists indexed by their collection ID.
*
* Order is preserved.
*/
export const groupFilesByCollectionID = (files: EnteFile[]) =>
files.reduce((result, file) => {
const id = file.collectionID;
let cfs = result.get(id);
if (!cfs) result.set(id, (cfs = []));
cfs.push(file);
return result;
}, new Map<number, EnteFile[]>());

30
web/packages/new/photos/components/gallery/reducer.ts
View File

@@ -3,6 +3,11 @@ import {
isArchivedCollection,
isPinnedCollection,
} from "ente-gallery/services/magic-metadata";
import {
groupFilesByCollectionID,
sortFiles,
uniqueFilesByID,
} from "ente-gallery/utils/files";
import { collectionTypes, type Collection } from "ente-media/collection";
import type { EnteFile } from "ente-media/file";
import {
@@ -14,6 +19,8 @@ import {
createCollectionNameByID,
isHiddenCollection,
} from "ente-new/photos/services/collection";
import { getLatestVersionFiles } from "ente-new/photos/services/files";
import { type EnteTrashFile } from "ente-new/photos/services/trash";
import { splitByPredicate } from "ente-utils/array";
import { includes } from "ente-utils/type-guards";
import { t } from "i18next";
@@ -28,14 +35,6 @@ import {
type CollectionSummary,
type CollectionSummaryType,
} from "../../services/collection-summary";
import {
createFileCollectionIDs,
getLatestVersionFiles,
groupFilesByCollectionID,
sortFiles,
uniqueFilesByID,
type TrashedEnteFile,
} from "../../services/files";
import type { PeopleState, Person } from "../../services/ml/people";
import type { SearchSuggestion } from "../../services/search/types";
import type { FamilyData } from "../../services/user-details";
@@ -425,7 +424,7 @@ export type GalleryAction =
collections: Collection[];
normalFiles: EnteFile[];
hiddenFiles: EnteFile[];
trashedFiles: TrashedEnteFile[];
trashedFiles: EnteTrashFile[];
}
| {
type: "setCollections";
@@ -1251,6 +1250,19 @@ const deriveFavoriteFileIDs = (
return favoriteFileIDs;
};
/**
* Construct a map from file IDs to the list of collections (IDs) to which the
* file belongs.
*/
const createFileCollectionIDs = (files: EnteFile[]) =>
files.reduce((result, file) => {
const id = file.id;
let fs = result.get(id);
if (!fs) result.set(id, (fs = []));
fs.push(file.collectionID);
return result;
}, new Map<number, number[]>());
/**
* Compute normal (non-hidden) collection summaries from their dependencies.
*/

2
web/packages/new/photos/services/collections.ts
View File

@@ -11,8 +11,6 @@ import {
getLocalTrash,
getTrashedFiles,
TRASH,
} from "ente-new/photos/services/files";
import {
type EncryptedTrashItem,
type Trash,
} from "ente-new/photos/services/trash";

158
web/packages/new/photos/services/files.ts
View File

@@ -1,5 +1,4 @@
import { blobCache } from "ente-base/blob-cache";
import { dateFromEpochMicroseconds } from "ente-base/date";
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
import type { Collection } from "ente-media/collection";
@@ -8,10 +7,8 @@ import {
type EnteFile,
type RemoteEnteFile,
} from "ente-media/file";
import { fileCreationTime, metadataHash } from "ente-media/file-metadata";
import { type Trash } from "ente-new/photos/services/trash";
import { metadataHash } from "ente-media/file-metadata";
import HTTPService from "ente-shared/network/HTTPService";
import localForage from "ente-shared/storage/localForage";
import { getToken } from "ente-shared/storage/localStorage/helpers";
import {
getCollectionLastSyncTime,
@@ -151,130 +148,6 @@ const removeDeletedCollectionFiles = (
return files;
};
/**
* Sort the given list of {@link EnteFile}s in place.
*
* Like the JavaScript Array#sort, this method modifies the {@link files}
* argument. It sorts {@link files} in place, and then returns a reference to
* the same mutated array.
*
* By default, files are sorted so that the newest one is first. The optional
* {@link sortAsc} flag can be set to `true` to sort them so that the oldest one
* is first.
*/
export const sortFiles = (files: EnteFile[], sortAsc = false) => {
// Sort based on the time of creation time of the file.
//
// For files with same creation time, sort based on the time of last
// modification.
const factor = sortAsc ? -1 : 1;
return files.sort((a, b) => {
const at = fileCreationTime(a);
const bt = fileCreationTime(b);
return at == bt
? factor *
(b.metadata.modificationTime - a.metadata.modificationTime)
: factor * (bt - at);
});
};
/**
* [Note: Collection file]
*
* File IDs themselves are unique across all the files for the user (in fact,
* they're unique across all the files in an Ente instance).
*
* However, we can have multiple entries for the same file ID in our local
* database and/or remote responses because the unit of account is not file, but
* a "Collection File" – a collection and file pair.
*
* For example, if the same file is symlinked into two collections, then we will
* have two "Collection File" entries for it, both with the same file ID, but
* with different collection IDs.
*
* This function returns files such that only one of these entries is returned.
* The entry that is returned is arbitrary in general, this function just picks
* the first one for each unique file ID.
*
* If this function is invoked on a list on which {@link sortFiles} has already
* been called, which by default sorts such that the newest file is first, then
* this function's behaviour would be to return the newest file from among
* multiple files with the same ID but different collections.
*/
export const uniqueFilesByID = (files: EnteFile[]) => {
const seen = new Set<number>();
return files.filter(({ id }) => {
if (seen.has(id)) return false;
seen.add(id);
return true;
});
};
export const TRASH = "file-trash";
export async function getLocalTrash() {
const trash = (await localForage.getItem<Trash>(TRASH)) ?? [];
return trash;
}
export async function getLocalTrashedFiles() {
return getTrashedFiles(await getLocalTrash());
}
export type TrashedEnteFile = EnteFile & {
/**
* `true` if this file is in trash (i.e. it has been deleted by the user,
* and will be permanently deleted after 30 days of being moved to trash).
*/
isTrashed?: boolean;
/**
* If {@link isTrashed} is `true`, then {@link deleteBy} contains the epoch
* microseconds when this file will be permanently deleted.
*/
deleteBy?: number;
};
/**
* Return the date when the file will be deleted permanently. Only valid for
* files that are in the user's trash.
*
* This is a convenience wrapper over the {@link deleteBy} property of a file,
* converting that epoch microsecond value into a JavaScript date.
*/
export const enteFileDeletionDate = (file: TrashedEnteFile) =>
dateFromEpochMicroseconds(file.deleteBy);
export function getTrashedFiles(trash: Trash): TrashedEnteFile[] {
return sortTrashFiles(
trash.map((trashedFile) => ({
...trashedFile.file,
updationTime: trashedFile.updatedAt,
deleteBy: trashedFile.deleteBy,
isTrashed: true,
})),
);
}
/**
* Return the IDs of all the files that are part of the trash as per our local
* database.
*/
export const getLocalTrashFileIDs = () =>
getLocalTrash().then((trash) => new Set(trash.map((f) => f.file.id)));
const sortTrashFiles = (files: TrashedEnteFile[]) => {
return files.sort((a, b) => {
if (a.deleteBy === b.deleteBy) {
const at = fileCreationTime(a);
const bt = fileCreationTime(b);
return at == bt
? b.metadata.modificationTime - a.metadata.modificationTime
: bt - at;
}
return (a.deleteBy ?? 0) - (b.deleteBy ?? 0);
});
};
/**
* Clear cached thumbnails for existing files if the thumbnail data has changed.
*
@@ -295,7 +168,7 @@ const sortTrashFiles = (files: TrashedEnteFile[]) => {
*
* @param newFiles The {@link EnteFile}s which we got in the diff response.
*/
export const clearCachedThumbnailsIfChanged = async (
const clearCachedThumbnailsIfChanged = async (
existingFiles: EnteFile[],
newFiles: EnteFile[],
) => {
@@ -328,33 +201,6 @@ export const clearCachedThumbnailsIfChanged = async (
}
};
/**
* Segment the given {@link files} into lists indexed by their collection ID.
*
* Order is preserved.
*/
export const groupFilesByCollectionID = (files: EnteFile[]) =>
files.reduce((result, file) => {
const id = file.collectionID;
let cfs = result.get(id);
if (!cfs) result.set(id, (cfs = []));
cfs.push(file);
return result;
}, new Map<number, EnteFile[]>());
/**
* Construct a map from file IDs to the list of collections (IDs) to which the
* file belongs.
*/
export const createFileCollectionIDs = (files: EnteFile[]) =>
files.reduce((result, file) => {
const id = file.id;
let fs = result.get(id);
if (!fs) result.set(id, (fs = []));
fs.push(file.collectionID);
return result;
}, new Map<number, number[]>());
export function getLatestVersionFiles(files: EnteFile[]) {
const latestVersionFiles = new Map<string, EnteFile>();
files.forEach((file) => {

2
web/packages/new/photos/services/ml/worker.ts
View File

@@ -8,8 +8,8 @@ import type { ElectronMLWorker } from "ente-base/types/ipc";
import { isNetworkDownloadError } from "ente-gallery/services/download";
import type { ProcessableUploadItem } from "ente-gallery/services/upload";
import { fileLogID, type EnteFile } from "ente-media/file";
import { getLocalTrashFileIDs } from "ente-new/photos/services/trash";
import { wait } from "ente-utils/promise";
import { getLocalTrashFileIDs } from "../files";
import { savedFiles } from "../photos-fdb";
import {
createImageBitmapAndData,

2
web/packages/new/photos/services/search/index.ts
View File

@@ -1,9 +1,9 @@
import log from "ente-base/log";
import { ensureMasterKeyFromSession } from "ente-base/session";
import { ComlinkWorker } from "ente-base/worker/comlink-worker";
import { uniqueFilesByID } from "ente-gallery/utils/files";
import { FileType } from "ente-media/file-type";
import i18n, { t } from "i18next";
import { uniqueFilesByID } from "../files";
import { clipMatches, isMLEnabled, isMLSupported } from "../ml";
import type { NamedPerson } from "../ml/people";
import type {

55
web/packages/new/photos/services/trash.ts
View File

@@ -1,4 +1,6 @@
import type { EnteFile, RemoteEnteFile } from "ente-media/file";
import { fileCreationTime } from "ente-media/file-metadata";
import localForage from "ente-shared/storage/localForage";
export interface TrashItem extends Omit<EncryptedTrashItem, "file"> {
file: EnteFile;
@@ -29,3 +31,56 @@ export interface EncryptedTrashItem {
}
export type Trash = TrashItem[];
export const TRASH = "file-trash";
export async function getLocalTrash() {
const trash = (await localForage.getItem<Trash>(TRASH)) ?? [];
return trash;
}
export async function getLocalTrashedFiles() {
return getTrashedFiles(await getLocalTrash());
}
/**
* A file augmented with the date when it will be permanently deleted.
*/
export type EnteTrashFile = EnteFile & {
/**
* Timestamp (epoch microseconds) when this file, which is already in trash,
* will be permanently deleted.
*
* On being deleted by the user, files move to trash, will be permanently
* deleted after 30 days of being moved to trash)
*/
deleteBy?: number;
};
export const getTrashedFiles = (trash: Trash): EnteTrashFile[] =>
sortTrashFiles(
trash.map(({ file, updatedAt, deleteBy }) => ({
...file,
updationTime: updatedAt,
deleteBy,
})),
);
const sortTrashFiles = (files: EnteTrashFile[]) =>
files.sort((a, b) => {
if (a.deleteBy === b.deleteBy) {
const at = fileCreationTime(a);
const bt = fileCreationTime(b);
return at == bt
? b.metadata.modificationTime - a.metadata.modificationTime
: bt - at;
}
return (a.deleteBy ?? 0) - (b.deleteBy ?? 0);
});
/**
* Return the IDs of all the files that are part of the trash as per our local
* database.
*/
export const getLocalTrashFileIDs = () =>
getLocalTrash().then((trash) => new Set(trash.map((f) => f.file.id)));