From db95de88299eca2f35395fc4cdc0af91175a0ab2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:55:01 +0530 Subject: [PATCH 001/754] [mob][photos] Add cast pkg dependency --- mobile/pubspec.lock | 25 +++++++++++++++++++++++++ mobile/pubspec.yaml | 4 ++++ 2 files changed, 29 insertions(+) diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index bab6f37caa..3c9ff792c9 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -209,6 +209,15 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.1" + cast: + dependency: "direct main" + description: + path: "." + ref: multicast_version + resolved-ref: "1f39cd4d6efa9363e77b2439f0317bae0c92dda1" + url: "https://github.com/guyluz11/flutter_cast.git" + source: git + version: "2.0.9" characters: dependency: transitive description: @@ -1416,6 +1425,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.2" + multicast_dns: + dependency: transitive + description: + name: multicast_dns + sha256: "316cc47a958d4bd3c67bd238fe8b44fdfb6133bad89cb191c0c3bd3edb14e296" + url: "https://pub.dev" + source: hosted + version: "0.3.2+6" nested: dependency: transitive description: @@ -1729,6 +1746,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" + protobuf: + dependency: transitive + description: + name: protobuf + sha256: "68645b24e0716782e58948f8467fd42a880f255096a821f9e7d0ec625b00c84d" + url: "https://pub.dev" + source: hosted + version: "3.1.0" provider: dependency: "direct main" description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index f1575bdf8b..bc0d4ba2d7 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -27,6 +27,10 @@ dependencies: battery_info: ^1.1.1 bip39: ^1.0.6 cached_network_image: ^3.0.0 + cast: + git: + url: https://github.com/guyluz11/flutter_cast.git + ref: multicast_version chewie: git: url: https://github.com/ente-io/chewie.git From f645fff31c53c8f5dab6ad52102c589c67882e45 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:38:41 +0530 Subject: [PATCH 002/754] [mob][photos] Add hook to show cast devices --- mobile/ios/Runner/Info.plist | 9 +++++ .../gallery/gallery_app_bar_widget.dart | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/mobile/ios/Runner/Info.plist b/mobile/ios/Runner/Info.plist index 037996520e..cdbc237749 100644 --- a/mobile/ios/Runner/Info.plist +++ b/mobile/ios/Runner/Info.plist @@ -105,5 +105,14 @@ UIApplicationSupportsIndirectInputEvents + NSBonjourServices + + _googlecast._tcp + F5BCEC64._googlecast._tcp + + + NSLocalNetworkUsageDescription + ${PRODUCT_NAME} uses the local network to discover Cast-enabled devices on your WiFi + network. diff --git a/mobile/lib/ui/viewer/gallery/gallery_app_bar_widget.dart b/mobile/lib/ui/viewer/gallery/gallery_app_bar_widget.dart index 1026bd7fd4..aa09e49b29 100644 --- a/mobile/lib/ui/viewer/gallery/gallery_app_bar_widget.dart +++ b/mobile/lib/ui/viewer/gallery/gallery_app_bar_widget.dart @@ -2,6 +2,8 @@ import 'dart:async'; import 'dart:io'; import 'dart:math' as math; +import "package:cast/device.dart"; +import "package:cast/discovery_service.dart"; import "package:flutter/cupertino.dart"; import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; @@ -576,6 +578,9 @@ class _GalleryAppBarWidgetState extends State { ), ); } + if (widget.collection != null) { + actions.add(castWidget(context)); + } if (items.isNotEmpty) { actions.add( PopupMenuButton( @@ -642,6 +647,39 @@ class _GalleryAppBarWidgetState extends State { return actions; } + Widget castWidget(BuildContext context) { + return FutureBuilder>( + future: CastDiscoveryService().search(), + builder: (context, snapshot) { + if (snapshot.hasError) { + return Center( + child: Text( + 'Error: ${snapshot.error.toString()}', + ), + ); + } else if (!snapshot.hasData) { + return const Center( + child: CircularProgressIndicator(), + ); + } + + if (snapshot.data!.isEmpty) { + return const Text('No device'); + } + + return Column( + children: snapshot.data!.map((device) { + return Text(device.name); + + }).toList(), + ); + }, + ); + } + + Future _connectToYourApp( + BuildContext contect, CastDevice device,) async {} + Future onCleanUncategorizedClick(BuildContext buildContext) async { final actionResult = await showChoiceActionSheet( context, From 4b9446a9b0427dfec43e9a0d9e45d04cda34c85f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 09:41:10 +0530 Subject: [PATCH 003/754] Create a package to share code between photos and cast --- web/docs/dependencies.md | 9 ++++-- web/packages/media/.eslintrc.js | 3 ++ web/packages/media/README.md | 11 +++++++ web/packages/media/live-photo.ts | 52 ++++++++++++++++++++++++++++++++ web/packages/media/package.json | 9 ++++++ web/packages/media/tsconfig.json | 5 +++ web/yarn.lock | 2 +- 7 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 web/packages/media/.eslintrc.js create mode 100644 web/packages/media/README.md create mode 100644 web/packages/media/live-photo.ts create mode 100644 web/packages/media/package.json create mode 100644 web/packages/media/tsconfig.json diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index d0660bb3e8..1b36df12e7 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -110,7 +110,7 @@ with Next.js. For more details, see [translations.md](translations.md). -## Meta Frameworks +## Meta frameworks ### Next.js @@ -131,7 +131,12 @@ It is more lower level than Next, but the bells and whistles it doesn't have are the bells and whistles (and the accompanying complexity) that we don't need in some cases. -## Photos +## Media + +- "jszip" is used for reading zip files in JavaScript. Live photos are zip files + under the hood. + +## Photos app specific ### Misc diff --git a/web/packages/media/.eslintrc.js b/web/packages/media/.eslintrc.js new file mode 100644 index 0000000000..348075cd4f --- /dev/null +++ b/web/packages/media/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: ["@/build-config/eslintrc-next"], +}; diff --git a/web/packages/media/README.md b/web/packages/media/README.md new file mode 100644 index 0000000000..70d6424f29 --- /dev/null +++ b/web/packages/media/README.md @@ -0,0 +1,11 @@ +## @/media + +A package for sharing code between our apps that show media (photos, videos). + +Specifically, this is the intersection of code required by both the photos and +cast apps. + +### Packaging + +This (internal) package exports a React TypeScript library. We rely on the +importing project to transpile and bundle it. diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts new file mode 100644 index 0000000000..2d58c4c446 --- /dev/null +++ b/web/packages/media/live-photo.ts @@ -0,0 +1,52 @@ +import JSZip from "jszip"; + +class LivePhoto { + image: Uint8Array; + video: Uint8Array; + imageNameTitle: string; + videoNameTitle: string; +} + +export function getFileNameWithoutExtension(filename: string) { + const lastDotPosition = filename.lastIndexOf("."); + if (lastDotPosition === -1) return filename; + else return filename.slice(0, lastDotPosition); +} + +export function getFileExtensionWithDot(filename: string) { + const lastDotPosition = filename.lastIndexOf("."); + if (lastDotPosition === -1) return ""; + else return filename.slice(lastDotPosition); +} + +export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { + const originalName = getFileNameWithoutExtension(fileName); + const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); + + const livePhoto = new LivePhoto(); + for (const zipFilename in zip.files) { + if (zipFilename.startsWith("image")) { + livePhoto.imageNameTitle = + originalName + getFileExtensionWithDot(zipFilename); + livePhoto.image = await zip.files[zipFilename].async("uint8array"); + } else if (zipFilename.startsWith("video")) { + livePhoto.videoNameTitle = + originalName + getFileExtensionWithDot(zipFilename); + livePhoto.video = await zip.files[zipFilename].async("uint8array"); + } + } + return livePhoto; +}; + +export const encodeLivePhoto = async (livePhoto: LivePhoto) => { + const zip = new JSZip(); + zip.file( + "image" + getFileExtensionWithDot(livePhoto.imageNameTitle), + livePhoto.image, + ); + zip.file( + "video" + getFileExtensionWithDot(livePhoto.videoNameTitle), + livePhoto.video, + ); + return await zip.generateAsync({ type: "uint8array" }); +}; diff --git a/web/packages/media/package.json b/web/packages/media/package.json new file mode 100644 index 0000000000..7ab047317b --- /dev/null +++ b/web/packages/media/package.json @@ -0,0 +1,9 @@ +{ + "name": "@/media", + "version": "0.0.0", + "private": true, + "dependencies": { + "@/next": "*", + "jszip": "^3.10" + } +} diff --git a/web/packages/media/tsconfig.json b/web/packages/media/tsconfig.json new file mode 100644 index 0000000000..f29c348113 --- /dev/null +++ b/web/packages/media/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@/build-config/tsconfig-typecheck.json", + /* Typecheck all files with the given extensions (here or in subfolders) */ + "include": ["**/*.ts", "**/*.tsx"] +} diff --git a/web/yarn.lock b/web/yarn.lock index 11cc8b8e12..8461b3855c 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -3252,7 +3252,7 @@ jssha@~3.3.1: object.assign "^4.1.4" object.values "^1.1.6" -jszip@3.10.1: +jszip@3.10.1, jszip@^3.10: version "3.10.1" resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== From 3172104578324a0e4a01b7c5b03a9ca9bc744e66 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 09:49:59 +0530 Subject: [PATCH 004/754] Start using @/media's version --- web/apps/cast/package.json | 2 +- web/apps/cast/src/utils/file/index.ts | 19 +++++++++++++++++-- web/apps/photos/package.json | 2 +- web/apps/photos/src/types/upload/index.ts | 5 +++++ web/apps/photos/src/utils/file/index.ts | 11 +++++++---- web/yarn.lock | 2 +- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/web/apps/cast/package.json b/web/apps/cast/package.json index ee318ef619..2437c6c145 100644 --- a/web/apps/cast/package.json +++ b/web/apps/cast/package.json @@ -3,11 +3,11 @@ "version": "0.0.0", "private": true, "dependencies": { + "@/media": "*", "@/next": "*", "@ente/accounts": "*", "@ente/eslint-config": "*", "@ente/shared": "*", - "jszip": "3.10.1", "mime-types": "^2.1.35" } } diff --git a/web/apps/cast/src/utils/file/index.ts b/web/apps/cast/src/utils/file/index.ts index 4f6311cbdf..8d724feb52 100644 --- a/web/apps/cast/src/utils/file/index.ts +++ b/web/apps/cast/src/utils/file/index.ts @@ -1,8 +1,8 @@ +import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { FILE_TYPE, RAW_FORMATS } from "constants/file"; import CastDownloadManager from "services/castDownloadManager"; -import { decodeLivePhoto } from "services/livePhotoService"; import { getFileType } from "services/typeDetectionService"; import { EncryptedEnteFile, @@ -115,6 +115,18 @@ export function isRawFileFromFileName(fileName: string) { return false; } +/** + * [Note: File name for local EnteFile objects] + * + * The title property in a file's metadata is the original file's name. The + * metadata of a file cannot be edited. So if later on the file's name is + * changed, then the edit is stored in the `editedName` property of the public + * metadata of the file. + * + * This function merges these edits onto the file object that we use locally. + * Effectively, post this step, the file's metadata.title can be used in lieu of + * its filename. + */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { return files.map((file) => { if (file.pubMagicMetadata?.data.editedTime) { @@ -137,7 +149,10 @@ export const getPreviewableImage = async ( await CastDownloadManager.downloadFile(castToken, file), ).blob(); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto( + file.metadata.title, + fileBlob, + ); fileBlob = new Blob([livePhoto.image]); } const fileType = await getFileType( diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index 6ae109af16..4ade92263c 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -3,6 +3,7 @@ "version": "0.0.0", "private": true, "dependencies": { + "@/media": "*", "@/next": "*", "@date-io/date-fns": "^2.14.0", "@ente/accounts": "*", @@ -25,7 +26,6 @@ "hdbscan": "0.0.1-alpha.5", "heic-convert": "^2.0.0", "idb": "^7.1.1", - "jszip": "3.10.1", "leaflet": "^1.9.4", "leaflet-defaulticon-compatibility": "^0.1.1", "localforage": "^1.9.0", diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 0d38f6190f..3deab0ed79 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -24,6 +24,11 @@ export function isDataStream(object: any): object is DataStream { export type Logger = (message: string) => void; export interface Metadata { + /** + * The file name. + * + * See: [Note: File name for local EnteFile objects] + */ title: string; creationTime: number; modificationTime: number; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 785921cc91..f954016fb6 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,3 +1,4 @@ +import { decodeLivePhoto } from "@/media/live-photo"; import { convertBytesToHumanReadable } from "@/next/file"; import log from "@/next/log"; import type { Electron } from "@/next/types/ipc"; @@ -32,7 +33,6 @@ import { updateFilePublicMagicMetadata, } from "services/fileService"; import heicConversionService from "services/heicConversionService"; -import { decodeLivePhoto } from "services/livePhotoService"; import { getFileType } from "services/typeDetectionService"; import { updateFileCreationDateInEXIF } from "services/upload/exifService"; import { @@ -97,7 +97,10 @@ export async function downloadFile(file: EnteFile) { await DownloadManager.getFile(file), ).blob(); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto( + file.metadata.title, + fileBlob, + ); const image = new File([livePhoto.image], livePhoto.imageNameTitle); const imageType = await getFileType(image); const tempImageURL = URL.createObjectURL( @@ -355,7 +358,7 @@ async function getRenderableLivePhotoURL( fileBlob: Blob, forceConvert: boolean, ): Promise { - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); const getRenderableLivePhotoImageURL = async () => { try { @@ -813,7 +816,7 @@ async function downloadFileDesktop( if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { const fileBlob = await new Response(updatedStream).blob(); - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); const imageExportName = await safeFileName( downloadDir, livePhoto.imageNameTitle, diff --git a/web/yarn.lock b/web/yarn.lock index 8461b3855c..61d2cfeae1 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -3252,7 +3252,7 @@ jssha@~3.3.1: object.assign "^4.1.4" object.values "^1.1.6" -jszip@3.10.1, jszip@^3.10: +jszip@^3.10: version "3.10.1" resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== From e9bc465353c1a5bdd6de2eab9c9b8f9974d0fd1d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:40:37 +0530 Subject: [PATCH 005/754] [server] Add request object for copying files --- server/ente/collection.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/ente/collection.go b/server/ente/collection.go index 763d07b9b6..61c46860a4 100644 --- a/server/ente/collection.go +++ b/server/ente/collection.go @@ -103,6 +103,13 @@ type AddFilesRequest struct { Files []CollectionFileItem `json:"files" binding:"required"` } +// CopyFileSyncRequest is request object for creating copy of Files, and those copy to the destination collection +type CopyFileSyncRequest struct { + SrcCollectionID int64 `json:"srcCollectionID" binding:"required"` + DstCollection int64 `json:"dstCollectionID" binding:"required"` + Files []CollectionFileItem `json:"files" binding:"required"` +} + // RemoveFilesRequest represents a request to remove files from a collection type RemoveFilesRequest struct { CollectionID int64 `json:"collectionID" binding:"required"` From 637d830f199adf25c8f19280cac67ae2aa615403 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 11:48:15 +0530 Subject: [PATCH 006/754] Replace encodelivephoto --- .../photos/src/services/livePhotoService.ts | 13 ++++++++ .../src/services/upload/livePhotoService.ts | 2 +- web/packages/media/live-photo.ts | 30 +++++++++---------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/livePhotoService.ts b/web/apps/photos/src/services/livePhotoService.ts index 4d96e812cc..2fa11c2ca8 100644 --- a/web/apps/photos/src/services/livePhotoService.ts +++ b/web/apps/photos/src/services/livePhotoService.ts @@ -5,6 +5,9 @@ import { getFileNameWithoutExtension, } from "utils/file"; +/** + * An in-memory representation of a live photo + */ class LivePhoto { image: Uint8Array; video: Uint8Array; @@ -31,6 +34,16 @@ export const decodeLivePhoto = async (file: EnteFile, zipBlob: Blob) => { return livePhoto; }; +/** + * Return a binary serialized representation of a live photo. + * + * This function takes the (in-memory) image and video data from the + * {@link livePhoto} object, writes them to a zip file (using the respective + * filenames), and returns the {@link Uint8Array} that represent the bytes of + * this zip file. + * + * @param livePhoto The in-mem photo to serialized. + */ export const encodeLivePhoto = async (livePhoto: LivePhoto) => { const zip = new JSZip(); zip.file( diff --git a/web/apps/photos/src/services/upload/livePhotoService.ts b/web/apps/photos/src/services/upload/livePhotoService.ts index 392b5b9c87..025c67e62c 100644 --- a/web/apps/photos/src/services/upload/livePhotoService.ts +++ b/web/apps/photos/src/services/upload/livePhotoService.ts @@ -4,7 +4,7 @@ import { CustomError } from "@ente/shared/error"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { LIVE_PHOTO_ASSET_SIZE_LIMIT } from "constants/upload"; -import { encodeLivePhoto } from "services/livePhotoService"; +import { encodeLivePhoto } from "@/media/live-photo"; import { getFileType } from "services/typeDetectionService"; import { ElectronFile, diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index 2d58c4c446..c5c1566e30 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -1,3 +1,4 @@ +import { nameAndExtension } from "@/next/file"; import JSZip from "jszip"; class LivePhoto { @@ -20,33 +21,30 @@ export function getFileExtensionWithDot(filename: string) { } export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { - const originalName = getFileNameWithoutExtension(fileName); + const [name] = nameAndExtension(fileName); const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); const livePhoto = new LivePhoto(); - for (const zipFilename in zip.files) { - if (zipFilename.startsWith("image")) { + for (const zipFileName in zip.files) { + if (zipFileName.startsWith("image")) { livePhoto.imageNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.image = await zip.files[zipFilename].async("uint8array"); - } else if (zipFilename.startsWith("video")) { + name + getFileExtensionWithDot(zipFileName); + livePhoto.image = await zip.files[zipFileName].async("uint8array"); + } else if (zipFileName.startsWith("video")) { livePhoto.videoNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.video = await zip.files[zipFilename].async("uint8array"); + name + getFileExtensionWithDot(zipFileName); + livePhoto.video = await zip.files[zipFileName].async("uint8array"); } } return livePhoto; }; export const encodeLivePhoto = async (livePhoto: LivePhoto) => { + const [, imageExt] = nameAndExtension(livePhoto.imageNameTitle); + const [, videoExt] = nameAndExtension(livePhoto.videoNameTitle); + const zip = new JSZip(); - zip.file( - "image" + getFileExtensionWithDot(livePhoto.imageNameTitle), - livePhoto.image, - ); - zip.file( - "video" + getFileExtensionWithDot(livePhoto.videoNameTitle), - livePhoto.video, - ); + zip.file(["image", imageExt].filter((x) => !!x).join("."), livePhoto.image); + zip.file(["video", videoExt].filter((x) => !!x).join("."), livePhoto.video); return await zip.generateAsync({ type: "uint8array" }); }; From 2d5ab044eebfa50508981e8d52cc9adc099cf75e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 11:55:07 +0530 Subject: [PATCH 007/754] Docs --- .../photos/src/services/livePhotoService.ts | 10 -------- web/packages/media/live-photo.ts | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/livePhotoService.ts b/web/apps/photos/src/services/livePhotoService.ts index 2fa11c2ca8..a4fa32de82 100644 --- a/web/apps/photos/src/services/livePhotoService.ts +++ b/web/apps/photos/src/services/livePhotoService.ts @@ -34,16 +34,6 @@ export const decodeLivePhoto = async (file: EnteFile, zipBlob: Blob) => { return livePhoto; }; -/** - * Return a binary serialized representation of a live photo. - * - * This function takes the (in-memory) image and video data from the - * {@link livePhoto} object, writes them to a zip file (using the respective - * filenames), and returns the {@link Uint8Array} that represent the bytes of - * this zip file. - * - * @param livePhoto The in-mem photo to serialized. - */ export const encodeLivePhoto = async (livePhoto: LivePhoto) => { const zip = new JSZip(); zip.file( diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index c5c1566e30..755bb40be8 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -20,6 +20,21 @@ export function getFileExtensionWithDot(filename: string) { else return filename.slice(lastDotPosition); } +/** + * Convert a binary serialized representation of a live photo to an in-memory + * {@link LivePhoto}. + * + * A live photo is a zip file containing two files - an image and a video. This + * functions reads that zip file (blob), and return separate bytes (and + * filenames) for the image and video parts. + * + * @param fileName The name of the overall live photo. Both the image and video + * parts of the decompressed live photo use this as their name, combined with + * their original extensions. + * + * @param zipBlob A blob contained the zipped data (i.e. the binary serialized + * live photo). + */ export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { const [name] = nameAndExtension(fileName); const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); @@ -39,6 +54,16 @@ export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { return livePhoto; }; +/** + * Return a binary serialized representation of a live photo. + * + * This function takes the (in-memory) image and video data from the + * {@link livePhoto} object, writes them to a zip file (using the respective + * filenames), and returns the {@link Uint8Array} that represent the bytes of + * this zip file. + * + * @param livePhoto The in-mem photo to serialized. + */ export const encodeLivePhoto = async (livePhoto: LivePhoto) => { const [, imageExt] = nameAndExtension(livePhoto.imageNameTitle); const [, videoExt] = nameAndExtension(livePhoto.videoNameTitle); From 27a2b087c76626aaf55db8620d0ed07be256a0b9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 11:59:50 +0530 Subject: [PATCH 008/754] Elaborate --- web/packages/media/live-photo.ts | 26 +++++++------------------- web/packages/next/file.ts | 21 ++++++++++++++++++--- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index 755bb40be8..96be4f613b 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -1,4 +1,4 @@ -import { nameAndExtension } from "@/next/file"; +import { fileNameFromComponents, nameAndExtension } from "@/next/file"; import JSZip from "jszip"; class LivePhoto { @@ -8,18 +8,6 @@ class LivePhoto { videoNameTitle: string; } -export function getFileNameWithoutExtension(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return filename; - else return filename.slice(0, lastDotPosition); -} - -export function getFileExtensionWithDot(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return ""; - else return filename.slice(lastDotPosition); -} - /** * Convert a binary serialized representation of a live photo to an in-memory * {@link LivePhoto}. @@ -42,12 +30,12 @@ export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { const livePhoto = new LivePhoto(); for (const zipFileName in zip.files) { if (zipFileName.startsWith("image")) { - livePhoto.imageNameTitle = - name + getFileExtensionWithDot(zipFileName); + const [, imageExt] = nameAndExtension(zipFileName); + livePhoto.imageNameTitle = fileNameFromComponents([name, imageExt]); livePhoto.image = await zip.files[zipFileName].async("uint8array"); } else if (zipFileName.startsWith("video")) { - livePhoto.videoNameTitle = - name + getFileExtensionWithDot(zipFileName); + const [, videoExt] = nameAndExtension(zipFileName); + livePhoto.videoNameTitle = fileNameFromComponents([name, videoExt]); livePhoto.video = await zip.files[zipFileName].async("uint8array"); } } @@ -69,7 +57,7 @@ export const encodeLivePhoto = async (livePhoto: LivePhoto) => { const [, videoExt] = nameAndExtension(livePhoto.videoNameTitle); const zip = new JSZip(); - zip.file(["image", imageExt].filter((x) => !!x).join("."), livePhoto.image); - zip.file(["video", videoExt].filter((x) => !!x).join("."), livePhoto.video); + zip.file(fileNameFromComponents(["image", imageExt]), livePhoto.image); + zip.file(fileNameFromComponents(["video", videoExt]), livePhoto.video); return await zip.generateAsync({ type: "uint8array" }); }; diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index b69fece505..fae9a6d001 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -1,19 +1,34 @@ import type { ElectronFile } from "./types/file"; +/** + * The two parts of a file name - the name itself, and an (optional) extension. + * + * The extension does not include the dot. + */ +type FileNameComponents = [name: string, extension: string | undefined]; + /** * Split a filename into its components - the name itself, and the extension (if * any) - returning both. The dot is not included in either. * * For example, `foo-bar.png` will be split into ["foo-bar", "png"]. + * + * See {@link fileNameFromComponents} for the inverse operation. */ -export const nameAndExtension = ( - fileName: string, -): [string, string | undefined] => { +export const nameAndExtension = (fileName: string): FileNameComponents => { const i = fileName.lastIndexOf("."); if (i == -1) return [fileName, undefined]; else return [fileName.slice(0, i), fileName.slice(i + 1)]; }; +/** + * Construct a file name from its components (name and extension). + * + * Inverse of {@link nameAndExtension}. + */ +export const fileNameFromComponents = (components: FileNameComponents) => + components.filter((x) => !!x).join("."); + export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`; } From 871cb417d6324fc33b0323798f451454994acd6f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:03:45 +0530 Subject: [PATCH 009/754] Replace --- .../cast/src/services/livePhotoService.ts | 32 ------------- web/apps/photos/src/services/export/index.ts | 4 +- .../photos/src/services/export/migration.ts | 7 ++- .../photos/src/services/livePhotoService.ts | 48 ------------------- .../photos/src/utils/machineLearning/index.ts | 4 +- 5 files changed, 9 insertions(+), 86 deletions(-) delete mode 100644 web/apps/cast/src/services/livePhotoService.ts delete mode 100644 web/apps/photos/src/services/livePhotoService.ts diff --git a/web/apps/cast/src/services/livePhotoService.ts b/web/apps/cast/src/services/livePhotoService.ts deleted file mode 100644 index 789234bd3e..0000000000 --- a/web/apps/cast/src/services/livePhotoService.ts +++ /dev/null @@ -1,32 +0,0 @@ -import JSZip from "jszip"; -import { EnteFile } from "types/file"; -import { - getFileExtensionWithDot, - getFileNameWithoutExtension, -} from "utils/file"; - -class LivePhoto { - image: Uint8Array; - video: Uint8Array; - imageNameTitle: string; - videoNameTitle: string; -} - -export const decodeLivePhoto = async (file: EnteFile, zipBlob: Blob) => { - const originalName = getFileNameWithoutExtension(file.metadata.title); - const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); - - const livePhoto = new LivePhoto(); - for (const zipFilename in zip.files) { - if (zipFilename.startsWith("image")) { - livePhoto.imageNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.image = await zip.files[zipFilename].async("uint8array"); - } else if (zipFilename.startsWith("video")) { - livePhoto.videoNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.video = await zip.files[zipFilename].async("uint8array"); - } - } - return livePhoto; -}; diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 7d62798823..37b228f607 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -38,7 +38,7 @@ import { writeStream } from "utils/native-stream"; import { getAllLocalCollections } from "../collectionService"; import downloadManager from "../download"; import { getAllLocalFiles } from "../fileService"; -import { decodeLivePhoto } from "../livePhotoService"; +import { decodeLivePhoto } from "@/media/live-photo"; import { migrateExport } from "./migration"; /** Name of the JSON file in which we keep the state of the export. */ @@ -1017,7 +1017,7 @@ class ExportService { ) { const electron = ensureElectron(); const fileBlob = await new Response(fileStream).blob(); - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); const imageExportName = await safeFileName( collectionExportPath, livePhoto.imageNameTitle, diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index b90c12e1c2..8591437e2b 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -1,3 +1,4 @@ +import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; @@ -7,7 +8,6 @@ import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import downloadManager from "services/download"; import { getAllLocalFiles } from "services/fileService"; -import { decodeLivePhoto } from "services/livePhotoService"; import { Collection } from "types/collection"; import { CollectionExportNames, @@ -318,7 +318,10 @@ async function getFileExportNamesFromExportedFiles( if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { const fileStream = await downloadManager.getFile(file); const fileBlob = await new Response(fileStream).blob(); - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto( + file.metadata.title, + fileBlob, + ); const imageExportName = getUniqueFileExportNameForMigration( collectionPath, livePhoto.imageNameTitle, diff --git a/web/apps/photos/src/services/livePhotoService.ts b/web/apps/photos/src/services/livePhotoService.ts deleted file mode 100644 index a4fa32de82..0000000000 --- a/web/apps/photos/src/services/livePhotoService.ts +++ /dev/null @@ -1,48 +0,0 @@ -import JSZip from "jszip"; -import { EnteFile } from "types/file"; -import { - getFileExtensionWithDot, - getFileNameWithoutExtension, -} from "utils/file"; - -/** - * An in-memory representation of a live photo - */ -class LivePhoto { - image: Uint8Array; - video: Uint8Array; - imageNameTitle: string; - videoNameTitle: string; -} - -export const decodeLivePhoto = async (file: EnteFile, zipBlob: Blob) => { - const originalName = getFileNameWithoutExtension(file.metadata.title); - const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); - - const livePhoto = new LivePhoto(); - for (const zipFilename in zip.files) { - if (zipFilename.startsWith("image")) { - livePhoto.imageNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.image = await zip.files[zipFilename].async("uint8array"); - } else if (zipFilename.startsWith("video")) { - livePhoto.videoNameTitle = - originalName + getFileExtensionWithDot(zipFilename); - livePhoto.video = await zip.files[zipFilename].async("uint8array"); - } - } - return livePhoto; -}; - -export const encodeLivePhoto = async (livePhoto: LivePhoto) => { - const zip = new JSZip(); - zip.file( - "image" + getFileExtensionWithDot(livePhoto.imageNameTitle), - livePhoto.image, - ); - zip.file( - "video" + getFileExtensionWithDot(livePhoto.videoNameTitle), - livePhoto.video, - ); - return await zip.generateAsync({ type: "uint8array" }); -}; diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index 2c199981a1..6a5984a4c9 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -1,9 +1,9 @@ +import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { FILE_TYPE } from "constants/file"; import PQueue from "p-queue"; import DownloadManager from "services/download"; import { getLocalFiles } from "services/fileService"; -import { decodeLivePhoto } from "services/livePhotoService"; import { EnteFile } from "types/file"; import { Dimensions } from "types/image"; import { @@ -134,7 +134,7 @@ async function getOriginalConvertedFile(file: EnteFile, queue?: PQueue) { if (file.metadata.fileType === FILE_TYPE.IMAGE) { return await getRenderableImage(file.metadata.title, fileBlob); } else { - const livePhoto = await decodeLivePhoto(file, fileBlob); + const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); return await getRenderableImage( livePhoto.imageNameTitle, new Blob([livePhoto.image]), From fb1d2c800a8ff7ef53017022852d85040d9d7cfc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:06:59 +0530 Subject: [PATCH 010/754] Dedup --- web/apps/cast/src/utils/file/index.ts | 12 ------- .../src/services/upload/livePhotoService.ts | 33 +++++++++++++++---- web/apps/photos/src/utils/file/index.ts | 12 ------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/web/apps/cast/src/utils/file/index.ts b/web/apps/cast/src/utils/file/index.ts index 8d724feb52..31ed5c577f 100644 --- a/web/apps/cast/src/utils/file/index.ts +++ b/web/apps/cast/src/utils/file/index.ts @@ -85,18 +85,6 @@ export async function decryptFile( } } -export function getFileNameWithoutExtension(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return filename; - else return filename.slice(0, lastDotPosition); -} - -export function getFileExtensionWithDot(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return ""; - else return filename.slice(lastDotPosition); -} - export function generateStreamFromArrayBuffer(data: Uint8Array) { return new ReadableStream({ async start(controller: ReadableStreamDefaultController) { diff --git a/web/apps/photos/src/services/upload/livePhotoService.ts b/web/apps/photos/src/services/upload/livePhotoService.ts index 025c67e62c..cb0ac93186 100644 --- a/web/apps/photos/src/services/upload/livePhotoService.ts +++ b/web/apps/photos/src/services/upload/livePhotoService.ts @@ -1,10 +1,10 @@ +import { encodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { Remote } from "comlink"; import { FILE_TYPE } from "constants/file"; import { LIVE_PHOTO_ASSET_SIZE_LIMIT } from "constants/upload"; -import { encodeLivePhoto } from "@/media/live-photo"; import { getFileType } from "services/typeDetectionService"; import { ElectronFile, @@ -14,12 +14,6 @@ import { LivePhotoAssets, ParsedMetadataJSONMap, } from "types/upload"; -import { - getFileExtensionWithDot, - getFileNameWithoutExtension, - isImageOrVideo, - splitFilenameAndExtension, -} from "utils/file"; import { getFileTypeFromExtensionForLivePhotoClustering } from "utils/file/livePhoto"; import { getUint8ArrayView } from "../readerService"; import { extractFileMetadata } from "./fileService"; @@ -304,3 +298,28 @@ function removePotentialLivePhotoSuffix( return filenameWithoutExtension; } } + +function getFileNameWithoutExtension(filename: string) { + const lastDotPosition = filename.lastIndexOf("."); + if (lastDotPosition === -1) return filename; + else return filename.slice(0, lastDotPosition); +} + +function getFileExtensionWithDot(filename: string) { + const lastDotPosition = filename.lastIndexOf("."); + if (lastDotPosition === -1) return ""; + else return filename.slice(lastDotPosition); +} + +function splitFilenameAndExtension(filename: string): [string, string] { + const lastDotPosition = filename.lastIndexOf("."); + if (lastDotPosition === -1) return [filename, null]; + else + return [ + filename.slice(0, lastDotPosition), + filename.slice(lastDotPosition + 1), + ]; +} + +const isImageOrVideo = (fileType: FILE_TYPE) => + [FILE_TYPE.IMAGE, FILE_TYPE.VIDEO].includes(fileType); diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index f954016fb6..c1ee834482 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -250,18 +250,6 @@ export async function decryptFile( } } -export function getFileNameWithoutExtension(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return filename; - else return filename.slice(0, lastDotPosition); -} - -export function getFileExtensionWithDot(filename: string) { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return ""; - else return filename.slice(lastDotPosition); -} - export function splitFilenameAndExtension(filename: string): [string, string] { const lastDotPosition = filename.lastIndexOf("."); if (lastDotPosition === -1) return [filename, null]; From 652be207be26c8033492bd7aebedd318043e1a80 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:10:59 +0530 Subject: [PATCH 011/754] Update another place --- .../PhotoViewer/FileInfo/RenderFileName.tsx | 11 +++-------- web/apps/photos/src/services/export/migration.ts | 2 +- web/apps/photos/src/utils/ffmpeg/index.ts | 10 ---------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx index 74ae87380f..1bee86c25a 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderFileName.tsx @@ -1,3 +1,4 @@ +import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { FlexWrapper } from "@ente/shared/components/Container"; import PhotoOutlined from "@mui/icons-material/PhotoOutlined"; @@ -7,11 +8,7 @@ import { FILE_TYPE } from "constants/file"; import { useEffect, useState } from "react"; import { EnteFile } from "types/file"; import { makeHumanReadableStorage } from "utils/billing"; -import { - changeFileName, - splitFilenameAndExtension, - updateExistingFilePubMetadata, -} from "utils/file"; +import { changeFileName, updateExistingFilePubMetadata } from "utils/file"; import { FileNameEditDialog } from "./FileNameEditDialog"; import InfoItem from "./InfoItem"; @@ -65,9 +62,7 @@ export function RenderFileName({ const [extension, setExtension] = useState(); useEffect(() => { - const [filename, extension] = splitFilenameAndExtension( - file.metadata.title, - ); + const [filename, extension] = nameAndExtension(file.metadata.title); setFilename(filename); setExtension(extension); }, [file]); diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 8591437e2b..0403f93f7e 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -21,11 +21,11 @@ import { } from "types/export"; import { EnteFile } from "types/file"; import { getNonEmptyPersonalCollections } from "utils/collection"; -import { splitFilenameAndExtension } from "utils/ffmpeg"; import { getIDBasedSortedFiles, getPersonalFiles, mergeMetadata, + splitFilenameAndExtension, } from "utils/file"; import { safeDirectoryName, diff --git a/web/apps/photos/src/utils/ffmpeg/index.ts b/web/apps/photos/src/utils/ffmpeg/index.ts index 1b3445976a..8a4332a7fd 100644 --- a/web/apps/photos/src/utils/ffmpeg/index.ts +++ b/web/apps/photos/src/utils/ffmpeg/index.ts @@ -65,13 +65,3 @@ function parseCreationTime(creationTime: string) { } return dateTime; } - -export function splitFilenameAndExtension(filename: string): [string, string] { - const lastDotPosition = filename.lastIndexOf("."); - if (lastDotPosition === -1) return [filename, null]; - else - return [ - filename.slice(0, lastDotPosition), - filename.slice(lastDotPosition + 1), - ]; -} From 184ba91a2d4ce1d070f2212c4b177683877e6a93 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:23:50 +0530 Subject: [PATCH 012/754] Refactor --- web/apps/cast/src/utils/file/index.ts | 4 +- web/apps/photos/src/services/export/index.ts | 24 ++++---- .../photos/src/services/export/migration.ts | 6 +- .../src/services/upload/livePhotoService.ts | 12 ++-- web/apps/photos/src/utils/file/index.ts | 27 ++++----- .../photos/src/utils/machineLearning/index.ts | 8 +-- web/packages/media/live-photo.ts | 58 +++++++++++++------ 7 files changed, 81 insertions(+), 58 deletions(-) diff --git a/web/apps/cast/src/utils/file/index.ts b/web/apps/cast/src/utils/file/index.ts index 31ed5c577f..60ec0e56e6 100644 --- a/web/apps/cast/src/utils/file/index.ts +++ b/web/apps/cast/src/utils/file/index.ts @@ -137,11 +137,11 @@ export const getPreviewableImage = async ( await CastDownloadManager.downloadFile(castToken, file), ).blob(); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const livePhoto = await decodeLivePhoto( + const { imageData } = await decodeLivePhoto( file.metadata.title, fileBlob, ); - fileBlob = new Blob([livePhoto.image]); + fileBlob = new Blob([imageData]); } const fileType = await getFileType( new File([fileBlob], file.metadata.title), diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 37b228f607..882c36f9ba 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,3 +1,4 @@ +import { decodeLivePhoto } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; @@ -38,7 +39,6 @@ import { writeStream } from "utils/native-stream"; import { getAllLocalCollections } from "../collectionService"; import downloadManager from "../download"; import { getAllLocalFiles } from "../fileService"; -import { decodeLivePhoto } from "@/media/live-photo"; import { migrateExport } from "./migration"; /** Name of the JSON file in which we keep the state of the export. */ @@ -1015,18 +1015,18 @@ class ExportService { fileStream: ReadableStream, file: EnteFile, ) { - const electron = ensureElectron(); + const fs = ensureElectron().fs; const fileBlob = await new Response(fileStream).blob(); const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); const imageExportName = await safeFileName( collectionExportPath, - livePhoto.imageNameTitle, - electron.fs.exists, + livePhoto.imageFileName, + fs.exists, ); const videoExportName = await safeFileName( collectionExportPath, - livePhoto.videoNameTitle, - electron.fs.exists, + livePhoto.videoFileName, + fs.exists, ); const livePhotoExportName = getLivePhotoExportName( imageExportName, @@ -1038,7 +1038,9 @@ class ExportService { livePhotoExportName, ); try { - const imageStream = generateStreamFromArrayBuffer(livePhoto.image); + const imageStream = generateStreamFromArrayBuffer( + livePhoto.imageData, + ); await this.saveMetadataFile( collectionExportPath, imageExportName, @@ -1049,7 +1051,9 @@ class ExportService { imageStream, ); - const videoStream = generateStreamFromArrayBuffer(livePhoto.video); + const videoStream = generateStreamFromArrayBuffer( + livePhoto.videoData, + ); await this.saveMetadataFile( collectionExportPath, videoExportName, @@ -1061,9 +1065,7 @@ class ExportService { videoStream, ); } catch (e) { - await electron.fs.rm( - `${collectionExportPath}/${imageExportName}`, - ); + await fs.rm(`${collectionExportPath}/${imageExportName}`); throw e; } } catch (e) { diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 0403f93f7e..3f471b5399 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -318,18 +318,18 @@ async function getFileExportNamesFromExportedFiles( if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { const fileStream = await downloadManager.getFile(file); const fileBlob = await new Response(fileStream).blob(); - const livePhoto = await decodeLivePhoto( + const { imageFileName, videoFileName } = await decodeLivePhoto( file.metadata.title, fileBlob, ); const imageExportName = getUniqueFileExportNameForMigration( collectionPath, - livePhoto.imageNameTitle, + imageFileName, usedFilePaths, ); const videoExportName = getUniqueFileExportNameForMigration( collectionPath, - livePhoto.videoNameTitle, + videoFileName, usedFilePaths, ); fileExportName = getLivePhotoExportName( diff --git a/web/apps/photos/src/services/upload/livePhotoService.ts b/web/apps/photos/src/services/upload/livePhotoService.ts index cb0ac93186..c203c4d5f2 100644 --- a/web/apps/photos/src/services/upload/livePhotoService.ts +++ b/web/apps/photos/src/services/upload/livePhotoService.ts @@ -101,16 +101,16 @@ export async function readLivePhoto( }, ); - const image = await getUint8ArrayView(livePhotoAssets.image); + const imageData = await getUint8ArrayView(livePhotoAssets.image); - const video = await getUint8ArrayView(livePhotoAssets.video); + const videoData = await getUint8ArrayView(livePhotoAssets.video); return { filedata: await encodeLivePhoto({ - image, - video, - imageNameTitle: livePhotoAssets.image.name, - videoNameTitle: livePhotoAssets.video.name, + imageFileName: livePhotoAssets.image.name, + imageData, + videoFileName: livePhotoAssets.video.name, + videoData, }), thumbnail, hasStaticThumbnail, diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index c1ee834482..d4c9b81d8b 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -97,22 +97,20 @@ export async function downloadFile(file: EnteFile) { await DownloadManager.getFile(file), ).blob(); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - const livePhoto = await decodeLivePhoto( - file.metadata.title, - fileBlob, - ); - const image = new File([livePhoto.image], livePhoto.imageNameTitle); + const { imageFileName, imageData, videoFileName, videoData } = + await decodeLivePhoto(file.metadata.title, fileBlob); + const image = new File([imageData], imageFileName); const imageType = await getFileType(image); const tempImageURL = URL.createObjectURL( - new Blob([livePhoto.image], { type: imageType.mimeType }), + new Blob([imageData], { type: imageType.mimeType }), ); - const video = new File([livePhoto.video], livePhoto.videoNameTitle); + const video = new File([videoData], videoFileName); const videoType = await getFileType(video); const tempVideoURL = URL.createObjectURL( - new Blob([livePhoto.video], { type: videoType.mimeType }), + new Blob([videoData], { type: videoType.mimeType }), ); - downloadUsingAnchor(tempImageURL, livePhoto.imageNameTitle); - downloadUsingAnchor(tempVideoURL, livePhoto.videoNameTitle); + downloadUsingAnchor(tempImageURL, imageFileName); + downloadUsingAnchor(tempVideoURL, videoFileName); } else { const fileType = await getFileType( new File([fileBlob], file.metadata.title), @@ -350,9 +348,9 @@ async function getRenderableLivePhotoURL( const getRenderableLivePhotoImageURL = async () => { try { - const imageBlob = new Blob([livePhoto.image]); + const imageBlob = new Blob([livePhoto.imageData]); const convertedImageBlob = await getRenderableImage( - livePhoto.imageNameTitle, + livePhoto.imageFileName, imageBlob, ); @@ -365,10 +363,9 @@ async function getRenderableLivePhotoURL( const getRenderableLivePhotoVideoURL = async () => { try { - const videoBlob = new Blob([livePhoto.video]); - + const videoBlob = new Blob([livePhoto.videoData]); const convertedVideoBlob = await getPlayableVideo( - livePhoto.videoNameTitle, + livePhoto.videoFileName, videoBlob, forceConvert, true, diff --git a/web/apps/photos/src/utils/machineLearning/index.ts b/web/apps/photos/src/utils/machineLearning/index.ts index 6a5984a4c9..a89bccc4ca 100644 --- a/web/apps/photos/src/utils/machineLearning/index.ts +++ b/web/apps/photos/src/utils/machineLearning/index.ts @@ -134,11 +134,11 @@ async function getOriginalConvertedFile(file: EnteFile, queue?: PQueue) { if (file.metadata.fileType === FILE_TYPE.IMAGE) { return await getRenderableImage(file.metadata.title, fileBlob); } else { - const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); - return await getRenderableImage( - livePhoto.imageNameTitle, - new Blob([livePhoto.image]), + const { imageFileName, imageData } = await decodeLivePhoto( + file.metadata.title, + fileBlob, ); + return await getRenderableImage(imageFileName, new Blob([imageData])); } } diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts index 96be4f613b..16143ca138 100644 --- a/web/packages/media/live-photo.ts +++ b/web/packages/media/live-photo.ts @@ -1,11 +1,14 @@ import { fileNameFromComponents, nameAndExtension } from "@/next/file"; import JSZip from "jszip"; -class LivePhoto { - image: Uint8Array; - video: Uint8Array; - imageNameTitle: string; - videoNameTitle: string; +/** + * An in-memory representation of a live photo. + */ +interface LivePhoto { + imageFileName: string; + imageData: Uint8Array; + videoFileName: string; + videoData: Uint8Array; } /** @@ -23,23 +26,39 @@ class LivePhoto { * @param zipBlob A blob contained the zipped data (i.e. the binary serialized * live photo). */ -export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { +export const decodeLivePhoto = async ( + fileName: string, + zipBlob: Blob, +): Promise => { + let imageFileName, videoFileName: string | undefined; + let imageData, videoData: Uint8Array | undefined; + const [name] = nameAndExtension(fileName); const zip = await JSZip.loadAsync(zipBlob, { createFolders: true }); - const livePhoto = new LivePhoto(); for (const zipFileName in zip.files) { if (zipFileName.startsWith("image")) { const [, imageExt] = nameAndExtension(zipFileName); - livePhoto.imageNameTitle = fileNameFromComponents([name, imageExt]); - livePhoto.image = await zip.files[zipFileName].async("uint8array"); + imageFileName = fileNameFromComponents([name, imageExt]); + imageData = await zip.files[zipFileName]?.async("uint8array"); } else if (zipFileName.startsWith("video")) { const [, videoExt] = nameAndExtension(zipFileName); - livePhoto.videoNameTitle = fileNameFromComponents([name, videoExt]); - livePhoto.video = await zip.files[zipFileName].async("uint8array"); + videoFileName = fileNameFromComponents([name, videoExt]); + videoData = await zip.files[zipFileName]?.async("uint8array"); } } - return livePhoto; + + if (!imageFileName || !imageData) + throw new Error( + `Decoded live photo ${fileName} does not have an image`, + ); + + if (!videoFileName || !videoData) + throw new Error( + `Decoded live photo ${fileName} does not have an image`, + ); + + return { imageFileName, imageData, videoFileName, videoData }; }; /** @@ -52,12 +71,17 @@ export const decodeLivePhoto = async (fileName: string, zipBlob: Blob) => { * * @param livePhoto The in-mem photo to serialized. */ -export const encodeLivePhoto = async (livePhoto: LivePhoto) => { - const [, imageExt] = nameAndExtension(livePhoto.imageNameTitle); - const [, videoExt] = nameAndExtension(livePhoto.videoNameTitle); +export const encodeLivePhoto = async ({ + imageFileName, + imageData, + videoFileName, + videoData, +}: LivePhoto) => { + const [, imageExt] = nameAndExtension(imageFileName); + const [, videoExt] = nameAndExtension(videoFileName); const zip = new JSZip(); - zip.file(fileNameFromComponents(["image", imageExt]), livePhoto.image); - zip.file(fileNameFromComponents(["video", videoExt]), livePhoto.video); + zip.file(fileNameFromComponents(["image", imageExt]), imageData); + zip.file(fileNameFromComponents(["video", videoExt]), videoData); return await zip.generateAsync({ type: "uint8array" }); }; From df483b075527b45f95c9d425d2495408feddca32 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:37:18 +0530 Subject: [PATCH 013/754] lint --- web/docs/dependencies.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index 1b36df12e7..7dece3a370 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -133,8 +133,8 @@ some cases. ## Media -- "jszip" is used for reading zip files in JavaScript. Live photos are zip files - under the hood. +- "jszip" is used for reading zip files in JavaScript. Live photos are zip + files under the hood. ## Photos app specific From 75e07353bef88199c687be1da62847c0ce97c30f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 12:38:52 +0530 Subject: [PATCH 014/754] Thank you tsc --- web/apps/photos/src/utils/file/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index d4c9b81d8b..f65d36bd9f 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -801,21 +801,22 @@ async function downloadFileDesktop( if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { const fileBlob = await new Response(updatedStream).blob(); - const livePhoto = await decodeLivePhoto(file.metadata.title, fileBlob); + const { imageFileName, imageData, videoFileName, videoData } = + await decodeLivePhoto(file.metadata.title, fileBlob); const imageExportName = await safeFileName( downloadDir, - livePhoto.imageNameTitle, + imageFileName, fs.exists, ); - const imageStream = generateStreamFromArrayBuffer(livePhoto.image); + const imageStream = generateStreamFromArrayBuffer(imageData); await writeStream(`${downloadDir}/${imageExportName}`, imageStream); try { const videoExportName = await safeFileName( downloadDir, - livePhoto.videoNameTitle, + videoFileName, fs.exists, ); - const videoStream = generateStreamFromArrayBuffer(livePhoto.video); + const videoStream = generateStreamFromArrayBuffer(videoData); await writeStream(`${downloadDir}/${videoExportName}`, videoStream); } catch (e) { await fs.rm(`${downloadDir}/${imageExportName}`); From 4971099da938a1b5f1ecb8924332e6aecc9db298 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 13:41:10 +0530 Subject: [PATCH 015/754] Merge --- desktop/src/main/ipc.ts | 5 ++++ desktop/src/main/services/watch.ts | 31 ++++++++++++++++++++++- desktop/src/preload.ts | 5 ++++ web/apps/photos/src/services/watch.ts | 36 ++++++--------------------- web/packages/next/types/ipc.ts | 11 ++++++++ 5 files changed, 58 insertions(+), 30 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index a5de4514f9..3d4e15c992 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -55,6 +55,7 @@ import { } from "./services/upload"; import { addWatchMapping, + folderWatchesAndFilesTherein, getWatchMappings, removeWatchMapping, updateWatchMappingIgnoredFiles, @@ -238,6 +239,10 @@ export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => { removeWatchMapping(watcher, folderPath), ); + ipcMain.handle("folderWatchesAndFilesTherein", () => + folderWatchesAndFilesTherein(watcher), + ); + ipcMain.handle("getWatchMappings", () => getWatchMappings()); ipcMain.handle( diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index 1d466d4156..125d65bf9a 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -1,7 +1,13 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; -import { FolderWatch, WatchStoreType } from "../../types/ipc"; +import { + FolderWatch, + WatchStoreType, + type ElectronFile, +} from "../../types/ipc"; +import { isFolder } from "../fs"; import { watchStore } from "../stores/watch.store"; +import { getDirFiles } from "./fs"; export const addWatchMapping = async ( watcher: FSWatcher, @@ -99,3 +105,26 @@ export function getWatchMappings() { function setWatchMappings(watchMappings: WatchStoreType["mappings"]) { watchStore.set("mappings", watchMappings); } + +export const folderWatchesAndFilesTherein = async ( + watcher: FSWatcher, +): Promise<[watch: FolderWatch, files: ElectronFile[]][]> => { + const mappings = await getWatchMappings(); + + const activeMappings = []; + for (const mapping of mappings) { + const mappingExists = await isFolder(mapping.folderPath); + if (!mappingExists) { + await removeWatchMapping(watcher, mapping.folderPath); + } else { + activeMappings.push(mapping); + } + } + + return Promise.all( + activeMappings.map(async (mapping) => [ + mapping, + await getDirFiles(mapping.folderPath), + ]), + ); +}; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 2749fa50d2..a3deb1fae0 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -220,6 +220,10 @@ const addWatchMapping = ( const removeWatchMapping = (folderPath: string): Promise => ipcRenderer.invoke("removeWatchMapping", folderPath); +const folderWatchesAndFilesTherein = (): Promise< + [watch: FolderWatch, files: ElectronFile[]][] +> => ipcRenderer.invoke("folderWatchesAndFilesTherein"); + const getWatchMappings = (): Promise => ipcRenderer.invoke("getWatchMappings"); @@ -343,6 +347,7 @@ contextBridge.exposeInMainWorld("electron", { showUploadZipDialog, // - Watch + folderWatchesAndFilesTherein, registerWatcherFunctions, addWatchMapping, removeWatchMapping, diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index 2d5ef02287..7eaf391a19 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -70,25 +70,20 @@ class WatchFolderService { async getAndSyncDiffOfFiles() { try { - let mappings = await this.getWatchMappings(); - - if (!mappings?.length) { + const watchesAndFiles = + await ensureElectron().folderWatchesAndFilesTherein(); + if (!watchesAndFiles) { return; } - mappings = await this.filterOutDeletedMappings(mappings); - this.eventQueue = []; - for (const mapping of mappings) { - const filesOnDisk: ElectronFile[] = - await ensureElectron().getDirFiles(mapping.folderPath); - - this.uploadDiffOfFiles(mapping, filesOnDisk); - this.trashDiffOfFiles(mapping, filesOnDisk); + for (const [mapping, files] of watchesAndFiles) { + this.uploadDiffOfFiles(mapping, files); + this.trashDiffOfFiles(mapping, files); } } catch (e) { - log.error("error while getting and syncing diff of files", e); + log.error("Ignoring error while syncing watched folders", e); } } @@ -144,23 +139,6 @@ class WatchFolderService { } } - private async filterOutDeletedMappings( - mappings: WatchMapping[], - ): Promise { - const notDeletedMappings = []; - for (const mapping of mappings) { - const mappingExists = await ensureElectron().isFolder( - mapping.folderPath, - ); - if (!mappingExists) { - ensureElectron().removeWatchMapping(mapping.folderPath); - } else { - notDeletedMappings.push(mapping); - } - } - return notDeletedMappings; - } - pushEvent(event: EventQueueItem) { this.eventQueue.push(event); this.debouncedRunNextEvent(); diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 85986b6391..800befa7fb 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -284,6 +284,17 @@ export interface Electron { // - Watch + /** + * Get the latest state of the watched folders. + * + * We persist the folder watches that the user has setup. This function goes + * through that list, prunes any folders that don't exist on disk anymore, + * and for each, also returns a list of files that exist in that folder. + */ + folderWatchesAndFilesTherein: () => Promise< + [watch: FolderWatch, files: ElectronFile[]][] + >; + registerWatcherFunctions: ( addFile: (file: ElectronFile) => Promise, removeFile: (path: string) => Promise, From 00c400f68270d83ae185c178e2bd72ac6441869d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 14:19:11 +0530 Subject: [PATCH 016/754] Reword --- web/apps/photos/src/services/watch.ts | 143 +++++++++++++------------- 1 file changed, 73 insertions(+), 70 deletions(-) diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index 7eaf391a19..e6e5004ce0 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -5,6 +5,7 @@ import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; +import type { FolderWatch } from "@/next/types/ipc"; import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload"; import debounce from "debounce"; import uploadManager from "services/upload/uploadManager"; @@ -70,18 +71,18 @@ class WatchFolderService { async getAndSyncDiffOfFiles() { try { - const watchesAndFiles = - await ensureElectron().folderWatchesAndFilesTherein(); - if (!watchesAndFiles) { - return; - } + const electron = ensureElectron(); + const mappings = await electron.getWatchMappings(); + if (!mappings) return; this.eventQueue = []; + const { events, nonExistentFolderPaths } = + await syncWithDisk(mappings); + this.eventQueue = [...this.eventQueue, ...events]; + this.debouncedRunNextEvent(); - for (const [mapping, files] of watchesAndFiles) { - this.uploadDiffOfFiles(mapping, files); - this.trashDiffOfFiles(mapping, files); - } + for (const path of nonExistentFolderPaths) + electron.removeWatchMapping(path); } catch (e) { log.error("Ignoring error while syncing watched folders", e); } @@ -91,54 +92,6 @@ class WatchFolderService { return this.currentEvent?.folderPath === mapping.folderPath; } - private uploadDiffOfFiles( - mapping: WatchMapping, - filesOnDisk: ElectronFile[], - ) { - const filesToUpload = getValidFilesToUpload(filesOnDisk, mapping); - - if (filesToUpload.length > 0) { - for (const file of filesToUpload) { - const event: EventQueueItem = { - type: "upload", - collectionName: this.getCollectionNameForMapping( - mapping, - file.path, - ), - folderPath: mapping.folderPath, - files: [file], - }; - this.pushEvent(event); - } - } - } - - private trashDiffOfFiles( - mapping: WatchMapping, - filesOnDisk: ElectronFile[], - ) { - const filesToRemove = mapping.syncedFiles.filter((file) => { - return !filesOnDisk.find( - (electronFile) => electronFile.path === file.path, - ); - }); - - if (filesToRemove.length > 0) { - for (const file of filesToRemove) { - const event: EventQueueItem = { - type: "trash", - collectionName: this.getCollectionNameForMapping( - mapping, - file.path, - ), - folderPath: mapping.folderPath, - paths: [file.path], - }; - this.pushEvent(event); - } - } - } - pushEvent(event: EventQueueItem) { this.eventQueue.push(event); this.debouncedRunNextEvent(); @@ -547,10 +500,7 @@ class WatchFolderService { } return { - collectionName: this.getCollectionNameForMapping( - mapping, - filePath, - ), + collectionName: getCollectionNameForMapping(mapping, filePath), folderPath: mapping.folderPath, }; } catch (e) { @@ -558,15 +508,6 @@ class WatchFolderService { } } - private getCollectionNameForMapping( - mapping: WatchMapping, - filePath: string, - ) { - return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER - ? getParentFolderName(filePath) - : mapping.rootFolderName; - } - async selectFolder(): Promise { try { const folderPath = await ensureElectron().selectDirectory(); @@ -716,3 +657,65 @@ function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) { mapping.syncedFiles.find((f) => f.path === file.path) ); } + +/** + * Determine which events we need to process to synchronize the watched albums + * with the corresponding on disk folders. + * + * Also return a list of previously created folder watches for this there is no + * longer any no corresponding folder on disk. + */ +const syncWithDisk = async ( + mappings: FolderWatch[], +): Promise<{ + events: EventQueueItem[]; + nonExistentFolderPaths: string[]; +}> => { + const activeMappings = []; + const nonExistentFolderPaths: string[] = []; + + for (const mapping of mappings) { + const active = await electron.isFolder(mapping.folderPath); + if (!active) nonExistentFolderPaths.push(mapping.folderPath); + else activeMappings.push(mapping); + } + + const events: EventQueueItem[] = []; + + for (const mapping of activeMappings) { + const files = await electron.getDirFiles(mapping.folderPath); + + const filesToUpload = getValidFilesToUpload(files, mapping); + + for (const file of filesToUpload) + events.push({ + type: "upload", + collectionName: getCollectionNameForMapping(mapping, file.path), + folderPath: mapping.folderPath, + files: [file], + }); + + const filesToRemove = mapping.syncedFiles.filter((file) => { + return !files.find((f) => f.path === file.path); + }); + + for (const file of filesToRemove) + events.push({ + type: "trash", + collectionName: getCollectionNameForMapping(mapping, file.path), + folderPath: mapping.folderPath, + paths: [file.path], + }); + } + + return { events, nonExistentFolderPaths }; +}; + +const getCollectionNameForMapping = ( + mapping: WatchMapping, + filePath: string, +) => { + return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER + ? getParentFolderName(filePath) + : mapping.rootFolderName; +}; From ee895069234aae9cb3b9f50fed6da5fafe384def Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 14:27:48 +0530 Subject: [PATCH 017/754] Unlegacy isDir --- desktop/src/main/fs.ts | 8 +++---- desktop/src/main/ipc.ts | 13 +++-------- desktop/src/main/services/watch.ts | 31 +-------------------------- desktop/src/preload.ts | 18 ++++------------ web/apps/photos/src/services/watch.ts | 7 +++--- web/packages/next/types/ipc.ts | 9 +++++--- 6 files changed, 20 insertions(+), 66 deletions(-) diff --git a/desktop/src/main/fs.ts b/desktop/src/main/fs.ts index 36de710c34..2428d3a80c 100644 --- a/desktop/src/main/fs.ts +++ b/desktop/src/main/fs.ts @@ -22,10 +22,8 @@ export const fsReadTextFile = async (filePath: string) => export const fsWriteFile = (path: string, contents: string) => fs.writeFile(path, contents); -/* TODO: Audit below this */ - -export const isFolder = async (dirPath: string) => { +export const fsIsDir = async (dirPath: string) => { if (!existsSync(dirPath)) return false; - const stats = await fs.stat(dirPath); - return stats.isDirectory(); + const stat = await fs.stat(dirPath); + return stat.isDirectory(); }; diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 3d4e15c992..e307ad167d 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -19,13 +19,13 @@ import { } from "./dialogs"; import { fsExists, + fsIsDir, fsMkdirIfNeeded, fsReadTextFile, fsRename, fsRm, fsRmdir, fsWriteFile, - isFolder, } from "./fs"; import { logToDisk } from "./log"; import { @@ -55,7 +55,6 @@ import { } from "./services/upload"; import { addWatchMapping, - folderWatchesAndFilesTherein, getWatchMappings, removeWatchMapping, updateWatchMappingIgnoredFiles, @@ -133,6 +132,8 @@ export const attachIPCHandlers = () => { fsWriteFile(path, contents), ); + ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); + // - Conversion ipcMain.handle("convertToJPEG", (_, fileData, filename) => @@ -184,10 +185,6 @@ export const attachIPCHandlers = () => { ipcMain.handle("showUploadZipDialog", () => showUploadZipDialog()); - // - FS Legacy - - ipcMain.handle("isFolder", (_, dirPath: string) => isFolder(dirPath)); - // - Upload ipcMain.handle("getPendingUploads", () => getPendingUploads()); @@ -239,10 +236,6 @@ export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => { removeWatchMapping(watcher, folderPath), ); - ipcMain.handle("folderWatchesAndFilesTherein", () => - folderWatchesAndFilesTherein(watcher), - ); - ipcMain.handle("getWatchMappings", () => getWatchMappings()); ipcMain.handle( diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index 125d65bf9a..1d466d4156 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -1,13 +1,7 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; -import { - FolderWatch, - WatchStoreType, - type ElectronFile, -} from "../../types/ipc"; -import { isFolder } from "../fs"; +import { FolderWatch, WatchStoreType } from "../../types/ipc"; import { watchStore } from "../stores/watch.store"; -import { getDirFiles } from "./fs"; export const addWatchMapping = async ( watcher: FSWatcher, @@ -105,26 +99,3 @@ export function getWatchMappings() { function setWatchMappings(watchMappings: WatchStoreType["mappings"]) { watchStore.set("mappings", watchMappings); } - -export const folderWatchesAndFilesTherein = async ( - watcher: FSWatcher, -): Promise<[watch: FolderWatch, files: ElectronFile[]][]> => { - const mappings = await getWatchMappings(); - - const activeMappings = []; - for (const mapping of mappings) { - const mappingExists = await isFolder(mapping.folderPath); - if (!mappingExists) { - await removeWatchMapping(watcher, mapping.folderPath); - } else { - activeMappings.push(mapping); - } - } - - return Promise.all( - activeMappings.map(async (mapping) => [ - mapping, - await getDirFiles(mapping.folderPath), - ]), - ); -}; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index a3deb1fae0..9341f02f23 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -118,6 +118,9 @@ const fsReadTextFile = (path: string): Promise => const fsWriteFile = (path: string, contents: string): Promise => ipcRenderer.invoke("fsWriteFile", path, contents); +const fsIsDir = (dirPath: string): Promise => + ipcRenderer.invoke("fsIsDir", dirPath); + // - AUDIT below this // - Conversion @@ -220,10 +223,6 @@ const addWatchMapping = ( const removeWatchMapping = (folderPath: string): Promise => ipcRenderer.invoke("removeWatchMapping", folderPath); -const folderWatchesAndFilesTherein = (): Promise< - [watch: FolderWatch, files: ElectronFile[]][] -> => ipcRenderer.invoke("folderWatchesAndFilesTherein"); - const getWatchMappings = (): Promise => ipcRenderer.invoke("getWatchMappings"); @@ -239,11 +238,6 @@ const updateWatchMappingIgnoredFiles = ( ): Promise => ipcRenderer.invoke("updateWatchMappingIgnoredFiles", folderPath, files); -// - FS Legacy - -const isFolder = (dirPath: string): Promise => - ipcRenderer.invoke("isFolder", dirPath); - // - Upload const getPendingUploads = (): Promise<{ @@ -327,6 +321,7 @@ contextBridge.exposeInMainWorld("electron", { rm: fsRm, readTextFile: fsReadTextFile, writeFile: fsWriteFile, + isDir: fsIsDir, }, // - Conversion @@ -347,7 +342,6 @@ contextBridge.exposeInMainWorld("electron", { showUploadZipDialog, // - Watch - folderWatchesAndFilesTherein, registerWatcherFunctions, addWatchMapping, removeWatchMapping, @@ -355,10 +349,6 @@ contextBridge.exposeInMainWorld("electron", { updateWatchMappingSyncedFiles, updateWatchMappingIgnoredFiles, - // - FS legacy - // TODO: Move these into fs + document + rename if needed - isFolder, - // - Upload getPendingUploads, diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index e6e5004ce0..bb11fa67e1 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -538,8 +538,7 @@ class WatchFolderService { async isFolder(folderPath: string) { try { - const isFolder = await ensureElectron().isFolder(folderPath); - return isFolder; + return await ensureElectron().fs.isDir(folderPath); } catch (e) { log.error("error while checking if folder exists", e); } @@ -675,8 +674,8 @@ const syncWithDisk = async ( const nonExistentFolderPaths: string[] = []; for (const mapping of mappings) { - const active = await electron.isFolder(mapping.folderPath); - if (!active) nonExistentFolderPaths.push(mapping.folderPath); + const valid = await electron.fs.isDir(mapping.folderPath); + if (!valid) nonExistentFolderPaths.push(mapping.folderPath); else activeMappings.push(mapping); } diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 800befa7fb..348f815bb9 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -199,6 +199,12 @@ export interface Electron { * @param contents The string contents to write. */ writeFile: (path: string, contents: string) => Promise; + + /** + * Return true if there is an item at {@link dirPath}, and it is as + * directory. + */ + isDir: (dirPath: string) => Promise; }; /* @@ -321,9 +327,6 @@ export interface Electron { files: FolderWatch["ignoredFiles"], ) => Promise; - // - FS legacy - isFolder: (dirPath: string) => Promise; - // - Upload getPendingUploads: () => Promise<{ From 2051ccee46a1ff45e6426412992b6af10f8bd906 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 14:48:03 +0530 Subject: [PATCH 018/754] List files alternate --- desktop/src/main/fs.ts | 5 +++++ desktop/src/main/ipc.ts | 3 +++ desktop/src/preload.ts | 4 ++++ web/packages/next/types/ipc.ts | 12 ++++++++++++ 4 files changed, 24 insertions(+) diff --git a/desktop/src/main/fs.ts b/desktop/src/main/fs.ts index 2428d3a80c..ebf53487c9 100644 --- a/desktop/src/main/fs.ts +++ b/desktop/src/main/fs.ts @@ -27,3 +27,8 @@ export const fsIsDir = async (dirPath: string) => { const stat = await fs.stat(dirPath); return stat.isDirectory(); }; + +export const fsLsFiles = async (dirPath: string) => + (await fs.readdir(dirPath, { withFileTypes: true })) + .filter((e) => e.isFile()) + .map((e) => e.name); diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index e307ad167d..2e01b4a106 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -20,6 +20,7 @@ import { import { fsExists, fsIsDir, + fsLsFiles, fsMkdirIfNeeded, fsReadTextFile, fsRename, @@ -134,6 +135,8 @@ export const attachIPCHandlers = () => { ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); + ipcMain.handle("fsLsFiles", (_, dirPath: string) => fsLsFiles(dirPath)); + // - Conversion ipcMain.handle("convertToJPEG", (_, fileData, filename) => diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 9341f02f23..2c2677cf1f 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -121,6 +121,9 @@ const fsWriteFile = (path: string, contents: string): Promise => const fsIsDir = (dirPath: string): Promise => ipcRenderer.invoke("fsIsDir", dirPath); +const fsLsFiles = (dirPath: string): Promise => + ipcRenderer.invoke("fsLsFiles", dirPath); + // - AUDIT below this // - Conversion @@ -322,6 +325,7 @@ contextBridge.exposeInMainWorld("electron", { readTextFile: fsReadTextFile, writeFile: fsWriteFile, isDir: fsIsDir, + lsFiles: fsLsFiles, }, // - Conversion diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 348f815bb9..55e7fca3fc 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -205,6 +205,18 @@ export interface Electron { * directory. */ isDir: (dirPath: string) => Promise; + + /** + * Return a list of the names of the files in the given directory. + * + * Note: + * + * - This is not recursive, it will only return the names of direct + * children. + * + * - It will return only the names of files, not directories. + */ + lsFiles: (dirPath: string) => Promise; }; /* From aabb8848289d006b8559b270bb5ec87b51ff698d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:19:20 +0530 Subject: [PATCH 019/754] [server] Add validation logic for file copy --- server/pkg/controller/collection.go | 35 +++++++++++++++++++++++++++++ server/pkg/repo/collection.go | 24 ++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/server/pkg/controller/collection.go b/server/pkg/controller/collection.go index 15c06fa331..455e240194 100644 --- a/server/pkg/controller/collection.go +++ b/server/pkg/controller/collection.go @@ -464,6 +464,41 @@ func (c *CollectionController) isRemoveAllowed(ctx *gin.Context, actorUserID int return nil } +func (c *CollectionController) isCopyAllowed(ctx *gin.Context, actorUserID int64, req ente.CopyFileSyncRequest) error { + // verify that srcCollectionID is accessible by actorUserID + if _, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{ + CollectionID: req.SrcCollectionID, + ActorUserID: actorUserID, + }); err != nil { + return stacktrace.Propagate(err, "failed to verify srcCollection access") + } + // verify that dstCollectionID is owned by actorUserID + if _, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{ + CollectionID: req.DstCollection, + ActorUserID: actorUserID, + VerifyOwner: true, + }); err != nil { + return stacktrace.Propagate(err, "failed to ownership of the dstCollection access") + } + // verify that all FileIDs exists in the srcCollection + fileIDs := make([]int64, len(req.Files)) + for idx, file := range req.Files { + fileIDs[idx] = file.ID + } + if err := c.CollectionRepo.VerifyAllFileIDsExistsInCollection(ctx, req.SrcCollectionID, fileIDs); err != nil { + return stacktrace.Propagate(err, "failed to verify fileIDs in srcCollection") + } + dsMap, err := c.FileRepo.GetOwnerToFileIDsMap(ctx, fileIDs) + if err != nil { + return err + } + // verify that none of the file belongs to actorUserID + if _, ok := dsMap[actorUserID]; ok { + return ente.NewBadRequestWithMessage("can not copy files owned by actor") + } + return nil +} + // GetDiffV2 returns the changes in user's collections since a timestamp, along with hasMore bool flag. func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID int64, sinceTime int64) ([]ente.File, bool, error) { reqContextLogger := log.WithFields(log.Fields{ diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go index 16ae853244..1ab54903bf 100644 --- a/server/pkg/repo/collection.go +++ b/server/pkg/repo/collection.go @@ -374,6 +374,30 @@ func (repo *CollectionRepository) DoesFileExistInCollections(fileID int64, cIDs return exists, stacktrace.Propagate(err, "") } +// VerifyAllFileIDsExistsInCollection returns error if the fileIDs don't exist in the collection +func (repo *CollectionRepository) VerifyAllFileIDsExistsInCollection(ctx context.Context, cID int64, fileIDs []int64) error { + fileIdMap := make(map[int64]bool) + rows, err := repo.DB.QueryContext(ctx, `SELECT file_id FROM collection_files WHERE collection_id = $1 AND is_deleted = $2 AND file_id = ALL ($3)`, + cID, false, pq.Array(fileIDs)) + if err != nil { + return stacktrace.Propagate(err, "") + } + for rows.Next() { + var fileID int64 + if err := rows.Scan(&fileID); err != nil { + return stacktrace.Propagate(err, "") + } + fileIdMap[fileID] = true + } + // find fileIds that are not present in the collection + for _, fileID := range fileIDs { + if _, ok := fileIdMap[fileID]; !ok { + return stacktrace.Propagate(fmt.Errorf("fileID %d not found in collection %d", fileID, cID), "") + } + } + return nil +} + // GetCollectionShareeRole returns true if the collection is shared with the user func (repo *CollectionRepository) GetCollectionShareeRole(cID int64, userID int64) (*ente.CollectionParticipantRole, error) { var role *ente.CollectionParticipantRole From c124cde471364038da1936b2e24ea32d650ad6b3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:43:48 +0530 Subject: [PATCH 020/754] [server] Add basic scaffold for copying files --- server/cmd/museum/main.go | 6 ++++- server/pkg/api/file.go | 19 ++++++++++++++- server/pkg/controller/collection.go | 2 +- server/pkg/controller/file_copy/file_copy.go | 25 ++++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 server/pkg/controller/file_copy/file_copy.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index c451b8b9c0..aa50d8e6c0 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -5,6 +5,7 @@ import ( "database/sql" b64 "encoding/base64" "fmt" + "github.com/ente-io/museum/pkg/controller/file_copy" "net/http" "os" "os/signal" @@ -389,9 +390,11 @@ func main() { timeout.WithHandler(healthCheckHandler.PingDBStats), timeout.WithResponse(timeOutResponse), )) + fileCopyCtrl := &file_copy.FileCopyController{FileController: fileController, CollectionCtrl: collectionController, S3Config: s3Config} fileHandler := &api.FileHandler{ - Controller: fileController, + Controller: fileController, + FileCopyCtrl: fileCopyCtrl, } privateAPI.GET("/files/upload-urls", fileHandler.GetUploadURLs) privateAPI.GET("/files/multipart-upload-urls", fileHandler.GetMultipartUploadURLs) @@ -400,6 +403,7 @@ func main() { privateAPI.GET("/files/preview/:fileID", fileHandler.GetThumbnail) privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail) privateAPI.POST("/files", fileHandler.CreateOrUpdate) + privateAPI.POST("/files/copy", fileHandler.CopyFiles) privateAPI.PUT("/files/update", fileHandler.Update) privateAPI.POST("/files/trash", fileHandler.Trash) privateAPI.POST("/files/size", fileHandler.GetSize) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index a65b9e3833..6bda9ab31a 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -1,6 +1,7 @@ package api import ( + "github.com/ente-io/museum/pkg/controller/file_copy" "net/http" "os" "strconv" @@ -20,7 +21,8 @@ import ( // FileHandler exposes request handlers for all encrypted file related requests type FileHandler struct { - Controller *controller.FileController + Controller *controller.FileController + FileCopyCtrl *file_copy.FileCopyController } // DefaultMaxBatchSize is the default maximum API batch size unless specified otherwise @@ -58,6 +60,21 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { c.JSON(http.StatusOK, response) } +// CopyFiles copies files that are owned by another user +func (h *FileHandler) CopyFiles(c *gin.Context) { + var req ente.CopyFileSyncRequest + if err := c.ShouldBindJSON(&req); err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + response, err := h.FileCopyCtrl.CopyFiles(c, req) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, response) +} + // Update updates already existing file func (h *FileHandler) Update(c *gin.Context) { enteApp := auth.GetApp(c) diff --git a/server/pkg/controller/collection.go b/server/pkg/controller/collection.go index 455e240194..cdfbb40808 100644 --- a/server/pkg/controller/collection.go +++ b/server/pkg/controller/collection.go @@ -464,7 +464,7 @@ func (c *CollectionController) isRemoveAllowed(ctx *gin.Context, actorUserID int return nil } -func (c *CollectionController) isCopyAllowed(ctx *gin.Context, actorUserID int64, req ente.CopyFileSyncRequest) error { +func (c *CollectionController) IsCopyAllowed(ctx *gin.Context, actorUserID int64, req ente.CopyFileSyncRequest) error { // verify that srcCollectionID is accessible by actorUserID if _, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{ CollectionID: req.SrcCollectionID, diff --git a/server/pkg/controller/file_copy/file_copy.go b/server/pkg/controller/file_copy/file_copy.go new file mode 100644 index 0000000000..6f581c5865 --- /dev/null +++ b/server/pkg/controller/file_copy/file_copy.go @@ -0,0 +1,25 @@ +package file_copy + +import ( + "github.com/ente-io/museum/ente" + "github.com/ente-io/museum/pkg/controller" + "github.com/ente-io/museum/pkg/utils/auth" + "github.com/ente-io/museum/pkg/utils/s3config" + "github.com/gin-gonic/gin" +) + +type FileCopyController struct { + S3Config *s3config.S3Config + FileController *controller.FileController + CollectionCtrl *controller.CollectionController +} + +func (fc *FileCopyController) CopyFiles(c *gin.Context, req ente.CopyFileSyncRequest) (interface{}, error) { + userID := auth.GetUserID(c.Request.Header) + err := fc.CollectionCtrl.IsCopyAllowed(c, userID, req) + if err != nil { + return nil, err + } + return nil, ente.NewInternalError("yet to implement actual copy") + +} From 52c35108ca013a6b1ff58b778e96e4dbce6413be Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 15:51:51 +0530 Subject: [PATCH 021/754] WIP 1 --- desktop/src/main/fs.ts | 5 - desktop/src/main/ipc.ts | 4 +- desktop/src/main/services/watch.ts | 26 +++ desktop/src/preload.ts | 6 +- web/apps/photos/src/services/watch.ts | 150 +++++++++++------- .../photos/src/types/watchFolder/index.ts | 9 -- web/packages/next/types/ipc.ts | 4 +- 7 files changed, 122 insertions(+), 82 deletions(-) diff --git a/desktop/src/main/fs.ts b/desktop/src/main/fs.ts index ebf53487c9..2428d3a80c 100644 --- a/desktop/src/main/fs.ts +++ b/desktop/src/main/fs.ts @@ -27,8 +27,3 @@ export const fsIsDir = async (dirPath: string) => { const stat = await fs.stat(dirPath); return stat.isDirectory(); }; - -export const fsLsFiles = async (dirPath: string) => - (await fs.readdir(dirPath, { withFileTypes: true })) - .filter((e) => e.isFile()) - .map((e) => e.name); diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 2e01b4a106..555e51ab1a 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -20,7 +20,7 @@ import { import { fsExists, fsIsDir, - fsLsFiles, + fsListFiles, fsMkdirIfNeeded, fsReadTextFile, fsRename, @@ -135,7 +135,7 @@ export const attachIPCHandlers = () => { ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); - ipcMain.handle("fsLsFiles", (_, dirPath: string) => fsLsFiles(dirPath)); + ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath)); // - Conversion diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index 1d466d4156..c8cd5ec828 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -2,6 +2,32 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; import { FolderWatch, WatchStoreType } from "../../types/ipc"; import { watchStore } from "../stores/watch.store"; +import path from "node:path"; +import fs from "node:fs/promises"; + +/** + * Return the paths of all the files under the given directory (recursive). + * + * This function walks the directory tree starting at {@link dirPath}, and + * returns a list of the absolute paths of all the files that exist therein. It + * will recursively traverse into nested directories, and return the absolute + * paths of the files there too. + * + * The returned paths are guaranteed to use POSIX separators ('/'). + */ +export const findFiles = async (dirPath: string) => { + const items = await fs.readdir(dirPath, { withFileTypes: true }) + let paths: string[] = []; + for (const item of items) { + const itemPath = path.posix.join(dirPath, item.name); + if (item.isFile()) { + paths.push(itemPath) + } else if (item.isDirectory()) { + paths = [...paths, ...await findFiles(itemPath)] + } + } + return paths +} export const addWatchMapping = async ( watcher: FSWatcher, diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 2c2677cf1f..0a55e57001 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -121,8 +121,8 @@ const fsWriteFile = (path: string, contents: string): Promise => const fsIsDir = (dirPath: string): Promise => ipcRenderer.invoke("fsIsDir", dirPath); -const fsLsFiles = (dirPath: string): Promise => - ipcRenderer.invoke("fsLsFiles", dirPath); +const fsListFiles = (dirPath: string): Promise => + ipcRenderer.invoke("fsListFiles", dirPath); // - AUDIT below this @@ -325,7 +325,7 @@ contextBridge.exposeInMainWorld("electron", { readTextFile: fsReadTextFile, writeFile: fsWriteFile, isDir: fsIsDir, - lsFiles: fsLsFiles, + listFiles: fsListFiles, }, // - Conversion diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index bb11fa67e1..be4f95aa0a 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -12,19 +12,43 @@ import uploadManager from "services/upload/uploadManager"; import { Collection } from "types/collection"; import { EncryptedEnteFile } from "types/file"; import { ElectronFile, FileWithCollection } from "types/upload"; -import { - EventQueueItem, - WatchMapping, - WatchMappingSyncedFile, -} from "types/watchFolder"; +import { WatchMapping, WatchMappingSyncedFile } from "types/watchFolder"; import { groupFilesBasedOnCollectionID } from "utils/file"; import { isSystemFile } from "utils/upload"; import { removeFromCollection } from "./collectionService"; import { getLocalFiles } from "./fileService"; +/** + * A file system event encapsulates a change that has occurred on disk that + * needs us to take some action within Ente to synchronize with the user's + * (Ente) albums. + * + * Events get added in two ways: + * + * - When the app starts, it reads the current state of files on disk and + * compares that with its last known state to determine what all events it + * missed. This is easier than it sounds as we have only two events, add and + * remove. + * + * - When the app is running, it gets live notifications from our file system + * watcher (from the Node.js layer) about changes that have happened on disk, + * which the app then enqueues onto the event queue if they pertain to the + * files we're interested in. + */ +interface FSEvent { + /** The action to take */ + action: "upload" | "trash"; + /** The path of the root folder corresponding to the {@link FolderWatch}. */ + folderPath: string; + /** If applicable, the name of the (Ente) collection the file belongs to. */ + collectionName?: string; + /** The absolute path to the file under consideration. */ + filePath: string; +} + class WatchFolderService { - private eventQueue: EventQueueItem[] = []; - private currentEvent: EventQueueItem; + private eventQueue: FSEvent[] = []; + private currentEvent: FSEvent; private currentlySyncedMapping: WatchMapping; private trashingDirQueue: string[] = []; private isEventRunning: boolean = false; @@ -94,6 +118,7 @@ class WatchFolderService { pushEvent(event: EventQueueItem) { this.eventQueue.push(event); + log.info("FS event", event); this.debouncedRunNextEvent(); } @@ -566,55 +591,41 @@ const getParentFolderName = (filePath: string) => { }; async function diskFileAddedCallback(file: ElectronFile) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(file.path); + const collectionNameAndFolderPath = + await watchFolderService.getCollectionNameAndFolderPath(file.path); - if (!collectionNameAndFolderPath) { - return; - } - - const { collectionName, folderPath } = collectionNameAndFolderPath; - - const event: EventQueueItem = { - type: "upload", - collectionName, - folderPath, - files: [file], - }; - watchFolderService.pushEvent(event); - log.info( - `added (upload) to event queue, collectionName:${event.collectionName} folderPath:${event.folderPath}, filesCount: ${event.files.length}`, - ); - } catch (e) { - log.error("error while calling diskFileAddedCallback", e); + if (!collectionNameAndFolderPath) { + return; } + + const { collectionName, folderPath } = collectionNameAndFolderPath; + + const event: EventQueueItem = { + type: "upload", + collectionName, + folderPath, + path: file.path, + }; + watchFolderService.pushEvent(event); } async function diskFileRemovedCallback(filePath: string) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(filePath); + const collectionNameAndFolderPath = + await watchFolderService.getCollectionNameAndFolderPath(filePath); - if (!collectionNameAndFolderPath) { - return; - } - - const { collectionName, folderPath } = collectionNameAndFolderPath; - - const event: EventQueueItem = { - type: "trash", - collectionName, - folderPath, - paths: [filePath], - }; - watchFolderService.pushEvent(event); - log.info( - `added (trash) to event queue collectionName:${event.collectionName} folderPath:${event.folderPath} , pathsCount: ${event.paths.length}`, - ); - } catch (e) { - log.error("error while calling diskFileRemovedCallback", e); + if (!collectionNameAndFolderPath) { + return; } + + const { collectionName, folderPath } = collectionNameAndFolderPath; + + const event: EventQueueItem = { + type: "trash", + collectionName, + folderPath, + path: filePath, + }; + watchFolderService.pushEvent(event); } async function diskFolderRemovedCallback(folderPath: string) { @@ -682,34 +693,51 @@ const syncWithDisk = async ( const events: EventQueueItem[] = []; for (const mapping of activeMappings) { - const files = await electron.getDirFiles(mapping.folderPath); + const folderPath = mapping.folderPath; - const filesToUpload = getValidFilesToUpload(files, mapping); + const paths = (await electron.fs.listFiles(folderPath)) + // Filter out hidden files (files whose names begins with a dot) + .filter((n) => !n.startsWith(".")) + // Prepend folderPath to get the full path + .map((f) => `${folderPath}/${f}`); - for (const file of filesToUpload) + // Files that are on disk but not yet synced. + const pathsToUpload = paths.filter( + (path) => !isSyncedOrIgnoredPath(path, mapping), + ); + + for (const path of pathsToUpload) events.push({ type: "upload", - collectionName: getCollectionNameForMapping(mapping, file.path), - folderPath: mapping.folderPath, - files: [file], + collectionName: getCollectionNameForMapping(mapping, path), + folderPath, + filePath: path, }); - const filesToRemove = mapping.syncedFiles.filter((file) => { - return !files.find((f) => f.path === file.path); - }); + // Synced files that are no longer on disk + const pathsToRemove = mapping.syncedFiles.filter( + (file) => !paths.includes(file.path), + ); - for (const file of filesToRemove) + for (const path of pathsToRemove) events.push({ type: "trash", - collectionName: getCollectionNameForMapping(mapping, file.path), + collectionName: getCollectionNameForMapping(mapping, path), folderPath: mapping.folderPath, - paths: [file.path], + filePath: path, }); } return { events, nonExistentFolderPaths }; }; +function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) { + return ( + mapping.ignoredFiles.includes(path) || + mapping.syncedFiles.find((f) => f.path === path) + ); +} + const getCollectionNameForMapping = ( mapping: WatchMapping, filePath: string, diff --git a/web/apps/photos/src/types/watchFolder/index.ts b/web/apps/photos/src/types/watchFolder/index.ts index bd55704de9..dda243e557 100644 --- a/web/apps/photos/src/types/watchFolder/index.ts +++ b/web/apps/photos/src/types/watchFolder/index.ts @@ -1,5 +1,4 @@ import { UPLOAD_STRATEGY } from "constants/upload"; -import { ElectronFile } from "types/upload"; export interface WatchMappingSyncedFile { path: string; @@ -14,11 +13,3 @@ export interface WatchMapping { syncedFiles: WatchMappingSyncedFile[]; ignoredFiles: string[]; } - -export interface EventQueueItem { - type: "upload" | "trash"; - folderPath: string; - collectionName?: string; - paths?: string[]; - files?: ElectronFile[]; -} diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 55e7fca3fc..b14c8e2704 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -207,7 +207,7 @@ export interface Electron { isDir: (dirPath: string) => Promise; /** - * Return a list of the names of the files in the given directory. + * Return a list of the file names of the files in the given directory. * * Note: * @@ -216,7 +216,7 @@ export interface Electron { * * - It will return only the names of files, not directories. */ - lsFiles: (dirPath: string) => Promise; + listFiles: (dirPath: string) => Promise; }; /* From a22423d0397ff471bef4b7868d4b71be06e73ab6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 17:00:51 +0530 Subject: [PATCH 022/754] WIP 2 --- desktop/src/main/ipc.ts | 8 ++- desktop/src/main/services/watch.ts | 24 +++----- desktop/src/preload.ts | 10 ++-- web/apps/photos/src/services/watch.ts | 85 ++++++++++++++------------- web/packages/next/types/ipc.ts | 52 ++++++++++------ 5 files changed, 95 insertions(+), 84 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 555e51ab1a..bc41545f7c 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -20,7 +20,6 @@ import { import { fsExists, fsIsDir, - fsListFiles, fsMkdirIfNeeded, fsReadTextFile, fsRename, @@ -56,6 +55,7 @@ import { } from "./services/upload"; import { addWatchMapping, + findFiles, getWatchMappings, removeWatchMapping, updateWatchMappingIgnoredFiles, @@ -135,8 +135,6 @@ export const attachIPCHandlers = () => { ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); - ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath)); - // - Conversion ipcMain.handle("convertToJPEG", (_, fileData, filename) => @@ -219,6 +217,10 @@ export const attachIPCHandlers = () => { export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => { // - Watch + ipcMain.handle("findFiles", (_, folderPath: string) => + findFiles(folderPath), + ); + ipcMain.handle( "addWatchMapping", ( diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index c8cd5ec828..53b50e31b5 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -1,33 +1,23 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; +import fs from "node:fs/promises"; +import path from "node:path"; import { FolderWatch, WatchStoreType } from "../../types/ipc"; import { watchStore } from "../stores/watch.store"; -import path from "node:path"; -import fs from "node:fs/promises"; -/** - * Return the paths of all the files under the given directory (recursive). - * - * This function walks the directory tree starting at {@link dirPath}, and - * returns a list of the absolute paths of all the files that exist therein. It - * will recursively traverse into nested directories, and return the absolute - * paths of the files there too. - * - * The returned paths are guaranteed to use POSIX separators ('/'). - */ export const findFiles = async (dirPath: string) => { - const items = await fs.readdir(dirPath, { withFileTypes: true }) + const items = await fs.readdir(dirPath, { withFileTypes: true }); let paths: string[] = []; for (const item of items) { const itemPath = path.posix.join(dirPath, item.name); if (item.isFile()) { - paths.push(itemPath) + paths.push(itemPath); } else if (item.isDirectory()) { - paths = [...paths, ...await findFiles(itemPath)] + paths = [...paths, ...(await findFiles(itemPath))]; } } - return paths -} + return paths; +}; export const addWatchMapping = async ( watcher: FSWatcher, diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 0a55e57001..53df1c968a 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -121,9 +121,6 @@ const fsWriteFile = (path: string, contents: string): Promise => const fsIsDir = (dirPath: string): Promise => ipcRenderer.invoke("fsIsDir", dirPath); -const fsListFiles = (dirPath: string): Promise => - ipcRenderer.invoke("fsListFiles", dirPath); - // - AUDIT below this // - Conversion @@ -194,6 +191,9 @@ const showUploadZipDialog = (): Promise<{ // - Watch +const findFiles = (folderPath: string): Promise => + ipcRenderer.invoke("findFiles", folderPath); + const registerWatcherFunctions = ( addFile: (file: ElectronFile) => Promise, removeFile: (path: string) => Promise, @@ -325,7 +325,6 @@ contextBridge.exposeInMainWorld("electron", { readTextFile: fsReadTextFile, writeFile: fsWriteFile, isDir: fsIsDir, - listFiles: fsListFiles, }, // - Conversion @@ -346,6 +345,9 @@ contextBridge.exposeInMainWorld("electron", { showUploadZipDialog, // - Watch + watch: { + findFiles, + }, registerWatcherFunctions, addWatchMapping, removeWatchMapping, diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index be4f95aa0a..c8ded2a42d 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -4,6 +4,7 @@ */ import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import type { FolderWatch } from "@/next/types/ipc"; import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload"; @@ -19,15 +20,15 @@ import { removeFromCollection } from "./collectionService"; import { getLocalFiles } from "./fileService"; /** - * A file system event encapsulates a change that has occurred on disk that - * needs us to take some action within Ente to synchronize with the user's - * (Ente) albums. + * A file system watch event encapsulates a change that has occurred on disk + * that needs us to take some action within Ente to synchronize with the user's + * Ente collections. * * Events get added in two ways: * * - When the app starts, it reads the current state of files on disk and * compares that with its last known state to determine what all events it - * missed. This is easier than it sounds as we have only two events, add and + * missed. This is easier than it sounds as we have only two events: add and * remove. * * - When the app is running, it gets live notifications from our file system @@ -35,20 +36,20 @@ import { getLocalFiles } from "./fileService"; * which the app then enqueues onto the event queue if they pertain to the * files we're interested in. */ -interface FSEvent { +interface WatchEvent { /** The action to take */ action: "upload" | "trash"; /** The path of the root folder corresponding to the {@link FolderWatch}. */ folderPath: string; - /** If applicable, the name of the (Ente) collection the file belongs to. */ + /** The name of the Ente collection the file belongs to. */ collectionName?: string; /** The absolute path to the file under consideration. */ filePath: string; } class WatchFolderService { - private eventQueue: FSEvent[] = []; - private currentEvent: FSEvent; + private eventQueue: WatchEvent[] = []; + private currentEvent: WatchEvent; private currentlySyncedMapping: WatchMapping; private trashingDirQueue: string[] = []; private isEventRunning: boolean = false; @@ -87,26 +88,26 @@ class WatchFolderService { this.setWatchFolderServiceIsRunning = setWatchFolderServiceIsRunning; this.setupWatcherFunctions(); - await this.getAndSyncDiffOfFiles(); + await this.syncWithDisk(); } catch (e) { log.error("error while initializing watch service", e); } } - async getAndSyncDiffOfFiles() { + private async syncWithDisk() { try { const electron = ensureElectron(); const mappings = await electron.getWatchMappings(); if (!mappings) return; this.eventQueue = []; - const { events, nonExistentFolderPaths } = - await syncWithDisk(mappings); + const { events, deletedFolderPaths } = await deduceEvents(mappings); this.eventQueue = [...this.eventQueue, ...events]; - this.debouncedRunNextEvent(); - for (const path of nonExistentFolderPaths) + for (const path of deletedFolderPaths) electron.removeWatchMapping(path); + + this.debouncedRunNextEvent(); } catch (e) { log.error("Ignoring error while syncing watched folders", e); } @@ -116,9 +117,9 @@ class WatchFolderService { return this.currentEvent?.folderPath === mapping.folderPath; } - pushEvent(event: EventQueueItem) { + private pushEvent(event: WatchEvent) { this.eventQueue.push(event); - log.info("FS event", event); + log.info("Watch event", event); this.debouncedRunNextEvent(); } @@ -145,7 +146,7 @@ class WatchFolderService { folderPath, uploadStrategy, ); - this.getAndSyncDiffOfFiles(); + this.syncWithDisk(); } catch (e) { log.error("error while adding watch mapping", e); } @@ -576,7 +577,7 @@ class WatchFolderService { resumePausedSync() { this.isPaused = false; - this.getAndSyncDiffOfFiles(); + this.syncWithDisk(); } } @@ -584,12 +585,6 @@ const watchFolderService = new WatchFolderService(); export default watchFolderService; -const getParentFolderName = (filePath: string) => { - const folderPath = filePath.substring(0, filePath.lastIndexOf("/")); - const folderName = folderPath.substring(folderPath.lastIndexOf("/") + 1); - return folderName; -}; - async function diskFileAddedCallback(file: ElectronFile) { const collectionNameAndFolderPath = await watchFolderService.getCollectionNameAndFolderPath(file.path); @@ -669,37 +664,35 @@ function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) { } /** - * Determine which events we need to process to synchronize the watched albums - * with the corresponding on disk folders. + * Determine which events we need to process to synchronize the watched on-disk + * folders to their corresponding collections. * - * Also return a list of previously created folder watches for this there is no - * longer any no corresponding folder on disk. + * Also return a list of previously created folder watches for which there is no + * longer any no corresponding directory on disk. */ -const syncWithDisk = async ( +const deduceEvents = async ( mappings: FolderWatch[], ): Promise<{ - events: EventQueueItem[]; - nonExistentFolderPaths: string[]; + events: WatchEvent[]; + deletedFolderPaths: string[]; }> => { const activeMappings = []; - const nonExistentFolderPaths: string[] = []; + const deletedFolderPaths: string[] = []; for (const mapping of mappings) { const valid = await electron.fs.isDir(mapping.folderPath); - if (!valid) nonExistentFolderPaths.push(mapping.folderPath); + if (!valid) deletedFolderPaths.push(mapping.folderPath); else activeMappings.push(mapping); } - const events: EventQueueItem[] = []; + const events: WatchEvent[] = []; for (const mapping of activeMappings) { const folderPath = mapping.folderPath; - const paths = (await electron.fs.listFiles(folderPath)) + const paths = (await electron.watch.findFiles(folderPath)) // Filter out hidden files (files whose names begins with a dot) - .filter((n) => !n.startsWith(".")) - // Prepend folderPath to get the full path - .map((f) => `${folderPath}/${f}`); + .filter((path) => !nameAndExtension(path)[0].startsWith(".")); // Files that are on disk but not yet synced. const pathsToUpload = paths.filter( @@ -708,7 +701,7 @@ const syncWithDisk = async ( for (const path of pathsToUpload) events.push({ - type: "upload", + action: "upload", collectionName: getCollectionNameForMapping(mapping, path), folderPath, filePath: path, @@ -728,7 +721,7 @@ const syncWithDisk = async ( }); } - return { events, nonExistentFolderPaths }; + return { events, deletedFolderPaths }; }; function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) { @@ -743,6 +736,16 @@ const getCollectionNameForMapping = ( filePath: string, ) => { return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER - ? getParentFolderName(filePath) + ? parentDirectoryName(filePath) : mapping.rootFolderName; }; + +const parentDirectoryName = (filePath: string) => { + const components = filePath.split("/"); + const parentName = components[components.length - 2]; + if (!parentName) + throw new Error( + `Unexpected file path without a parent folder: ${filePath}`, + ); + return parentName; +}; diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index b14c8e2704..5650d98748 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -205,18 +205,6 @@ export interface Electron { * directory. */ isDir: (dirPath: string) => Promise; - - /** - * Return a list of the file names of the files in the given directory. - * - * Note: - * - * - This is not recursive, it will only return the names of direct - * children. - * - * - It will return only the names of files, not directories. - */ - listFiles: (dirPath: string) => Promise; }; /* @@ -303,15 +291,32 @@ export interface Electron { // - Watch /** - * Get the latest state of the watched folders. + * Functions tailored for the folder watch functionality * - * We persist the folder watches that the user has setup. This function goes - * through that list, prunes any folders that don't exist on disk anymore, - * and for each, also returns a list of files that exist in that folder. + * [Note: Folder vs Directory in the context of FolderWatch-es] + * + * A note on terminology: The word "folder" is used to the top level root + * folder for which a {@link FolderWatch} has been added. This folder is + * also in 1-1 correspondence to be a directory on the user's disk. It can + * have other, nested directories too (which may or may not be getting + * mapped to separate Ente collections), but we'll not refer to these nested + * directories as folders - only the root of the tree, which the user + * dragged/dropped or selected to set up the folder watch, will be referred + * to as a folder when naming things. */ - folderWatchesAndFilesTherein: () => Promise< - [watch: FolderWatch, files: ElectronFile[]][] - >; + watch: { + /** + * Return the paths of all the files under the given folder. + * + * This function walks the directory tree starting at {@link folderPath} + * and returns a list of the absolute paths of all the files that exist + * therein. It will recursively traverse into nested directories, and + * return the absolute paths of the files there too. + * + * The returned paths are guaranteed to use POSIX separators ('/'). + */ + findFiles: (folderPath: string) => Promise; + }; registerWatcherFunctions: ( addFile: (file: ElectronFile) => Promise, @@ -327,6 +332,15 @@ export interface Electron { removeWatchMapping: (folderPath: string) => Promise; + /** + * TODO(MR): Outdated description + * Get the latest state of the watched folders. + * + * We persist the folder watches that the user has setup. This function goes + * through that list, prunes any folders that don't exist on disk anymore, + * and for each, also returns a list of files that exist in that folder. + */ + getWatchMappings: () => Promise; updateWatchMappingSyncedFiles: ( From 170ea0c9971a369a176962e29433c84c43e5b2ed Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 18:54:45 +0530 Subject: [PATCH 023/754] Cleanup --- .../photos/src/components/WatchFolder.tsx | 149 ++++++++---------- web/apps/photos/src/services/watch.ts | 95 +++++------ 2 files changed, 115 insertions(+), 129 deletions(-) diff --git a/web/apps/photos/src/components/WatchFolder.tsx b/web/apps/photos/src/components/WatchFolder.tsx index b5ff00b291..c0e7ab38ab 100644 --- a/web/apps/photos/src/components/WatchFolder.tsx +++ b/web/apps/photos/src/components/WatchFolder.tsx @@ -28,8 +28,8 @@ import { PICKED_UPLOAD_TYPE, UPLOAD_STRATEGY } from "constants/upload"; import { t } from "i18next"; import { AppContext } from "pages/_app"; import React, { useContext, useEffect, useState } from "react"; -import watchFolderService from "services/watch"; -import { WatchMapping } from "types/watchFolder"; +import watcher from "services/watch"; +import { WatchMapping as FolderWatch } from "types/watchFolder"; import { getImportSuggestion } from "utils/upload"; interface WatchFolderProps { @@ -38,7 +38,7 @@ interface WatchFolderProps { } export const WatchFolder: React.FC = ({ open, onClose }) => { - const [mappings, setMappings] = useState([]); + const [mappings, setMappings] = useState([]); const [inputFolderPath, setInputFolderPath] = useState(""); const [choiceModalOpen, setChoiceModalOpen] = useState(false); const appContext = useContext(AppContext); @@ -47,7 +47,7 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { useEffect(() => { if (!electron) return; - watchFolderService.getWatchMappings().then((m) => setMappings(m)); + watcher.getWatchMappings().then((m) => setMappings(m)); }, []); useEffect(() => { @@ -64,7 +64,7 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { for (let i = 0; i < folders.length; i++) { const folder: any = folders[i]; const path = (folder.path as string).replace(/\\/g, "/"); - if (await watchFolderService.isFolder(path)) { + if (await watcher.isFolder(path)) { await addFolderForWatching(path); } } @@ -91,7 +91,7 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { }; const handleFolderSelection = async () => { - const folderPath = await watchFolderService.selectFolder(); + const folderPath = await watcher.selectFolder(); if (folderPath) { await addFolderForWatching(folderPath); } @@ -102,19 +102,18 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { folderPath?: string, ) => { folderPath = folderPath || inputFolderPath; - await watchFolderService.addWatchMapping( + await watcher.addWatchMapping( folderPath.substring(folderPath.lastIndexOf("/") + 1), folderPath, uploadStrategy, ); setInputFolderPath(""); - setMappings(await watchFolderService.getWatchMappings()); + setMappings(await watcher.getWatchMappings()); }; - const handleRemoveWatchMapping = (mapping: WatchMapping) => { - watchFolderService - .mappingsAfterRemovingFolder(mapping.folderPath) - .then((ms) => setMappings(ms)); + const stopWatching = async (watch: FolderWatch) => { + await watcher.removeWatchForFolderPath(watch.folderPath); + setMappings(await watcher.getWatchMappings()); }; const closeChoiceModal = () => setChoiceModalOpen(false); @@ -144,9 +143,9 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { -