From a22423d0397ff471bef4b7868d4b71be06e73ab6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 17 Apr 2024 17:00:51 +0530 Subject: [PATCH] WIP 2 --- desktop/src/main/ipc.ts | 8 ++- desktop/src/main/services/watch.ts | 24 +++----- desktop/src/preload.ts | 10 ++-- web/apps/photos/src/services/watch.ts | 85 ++++++++++++++------------- web/packages/next/types/ipc.ts | 52 ++++++++++------ 5 files changed, 95 insertions(+), 84 deletions(-) diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 555e51ab1a..bc41545f7c 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -20,7 +20,6 @@ import { import { fsExists, fsIsDir, - fsListFiles, fsMkdirIfNeeded, fsReadTextFile, fsRename, @@ -56,6 +55,7 @@ import { } from "./services/upload"; import { addWatchMapping, + findFiles, getWatchMappings, removeWatchMapping, updateWatchMappingIgnoredFiles, @@ -135,8 +135,6 @@ export const attachIPCHandlers = () => { ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); - ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath)); - // - Conversion ipcMain.handle("convertToJPEG", (_, fileData, filename) => @@ -219,6 +217,10 @@ export const attachIPCHandlers = () => { export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => { // - Watch + ipcMain.handle("findFiles", (_, folderPath: string) => + findFiles(folderPath), + ); + ipcMain.handle( "addWatchMapping", ( diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index c8cd5ec828..53b50e31b5 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -1,33 +1,23 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; +import fs from "node:fs/promises"; +import path from "node:path"; import { FolderWatch, WatchStoreType } from "../../types/ipc"; import { watchStore } from "../stores/watch.store"; -import path from "node:path"; -import fs from "node:fs/promises"; -/** - * Return the paths of all the files under the given directory (recursive). - * - * This function walks the directory tree starting at {@link dirPath}, and - * returns a list of the absolute paths of all the files that exist therein. It - * will recursively traverse into nested directories, and return the absolute - * paths of the files there too. - * - * The returned paths are guaranteed to use POSIX separators ('/'). - */ export const findFiles = async (dirPath: string) => { - const items = await fs.readdir(dirPath, { withFileTypes: true }) + const items = await fs.readdir(dirPath, { withFileTypes: true }); let paths: string[] = []; for (const item of items) { const itemPath = path.posix.join(dirPath, item.name); if (item.isFile()) { - paths.push(itemPath) + paths.push(itemPath); } else if (item.isDirectory()) { - paths = [...paths, ...await findFiles(itemPath)] + paths = [...paths, ...(await findFiles(itemPath))]; } } - return paths -} + return paths; +}; export const addWatchMapping = async ( watcher: FSWatcher, diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 0a55e57001..53df1c968a 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -121,9 +121,6 @@ const fsWriteFile = (path: string, contents: string): Promise => const fsIsDir = (dirPath: string): Promise => ipcRenderer.invoke("fsIsDir", dirPath); -const fsListFiles = (dirPath: string): Promise => - ipcRenderer.invoke("fsListFiles", dirPath); - // - AUDIT below this // - Conversion @@ -194,6 +191,9 @@ const showUploadZipDialog = (): Promise<{ // - Watch +const findFiles = (folderPath: string): Promise => + ipcRenderer.invoke("findFiles", folderPath); + const registerWatcherFunctions = ( addFile: (file: ElectronFile) => Promise, removeFile: (path: string) => Promise, @@ -325,7 +325,6 @@ contextBridge.exposeInMainWorld("electron", { readTextFile: fsReadTextFile, writeFile: fsWriteFile, isDir: fsIsDir, - listFiles: fsListFiles, }, // - Conversion @@ -346,6 +345,9 @@ contextBridge.exposeInMainWorld("electron", { showUploadZipDialog, // - Watch + watch: { + findFiles, + }, registerWatcherFunctions, addWatchMapping, removeWatchMapping, diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index be4f95aa0a..c8ded2a42d 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -4,6 +4,7 @@ */ import { ensureElectron } from "@/next/electron"; +import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import type { FolderWatch } from "@/next/types/ipc"; import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload"; @@ -19,15 +20,15 @@ import { removeFromCollection } from "./collectionService"; import { getLocalFiles } from "./fileService"; /** - * A file system event encapsulates a change that has occurred on disk that - * needs us to take some action within Ente to synchronize with the user's - * (Ente) albums. + * A file system watch event encapsulates a change that has occurred on disk + * that needs us to take some action within Ente to synchronize with the user's + * Ente collections. * * Events get added in two ways: * * - When the app starts, it reads the current state of files on disk and * compares that with its last known state to determine what all events it - * missed. This is easier than it sounds as we have only two events, add and + * missed. This is easier than it sounds as we have only two events: add and * remove. * * - When the app is running, it gets live notifications from our file system @@ -35,20 +36,20 @@ import { getLocalFiles } from "./fileService"; * which the app then enqueues onto the event queue if they pertain to the * files we're interested in. */ -interface FSEvent { +interface WatchEvent { /** The action to take */ action: "upload" | "trash"; /** The path of the root folder corresponding to the {@link FolderWatch}. */ folderPath: string; - /** If applicable, the name of the (Ente) collection the file belongs to. */ + /** The name of the Ente collection the file belongs to. */ collectionName?: string; /** The absolute path to the file under consideration. */ filePath: string; } class WatchFolderService { - private eventQueue: FSEvent[] = []; - private currentEvent: FSEvent; + private eventQueue: WatchEvent[] = []; + private currentEvent: WatchEvent; private currentlySyncedMapping: WatchMapping; private trashingDirQueue: string[] = []; private isEventRunning: boolean = false; @@ -87,26 +88,26 @@ class WatchFolderService { this.setWatchFolderServiceIsRunning = setWatchFolderServiceIsRunning; this.setupWatcherFunctions(); - await this.getAndSyncDiffOfFiles(); + await this.syncWithDisk(); } catch (e) { log.error("error while initializing watch service", e); } } - async getAndSyncDiffOfFiles() { + private async syncWithDisk() { try { const electron = ensureElectron(); const mappings = await electron.getWatchMappings(); if (!mappings) return; this.eventQueue = []; - const { events, nonExistentFolderPaths } = - await syncWithDisk(mappings); + const { events, deletedFolderPaths } = await deduceEvents(mappings); this.eventQueue = [...this.eventQueue, ...events]; - this.debouncedRunNextEvent(); - for (const path of nonExistentFolderPaths) + for (const path of deletedFolderPaths) electron.removeWatchMapping(path); + + this.debouncedRunNextEvent(); } catch (e) { log.error("Ignoring error while syncing watched folders", e); } @@ -116,9 +117,9 @@ class WatchFolderService { return this.currentEvent?.folderPath === mapping.folderPath; } - pushEvent(event: EventQueueItem) { + private pushEvent(event: WatchEvent) { this.eventQueue.push(event); - log.info("FS event", event); + log.info("Watch event", event); this.debouncedRunNextEvent(); } @@ -145,7 +146,7 @@ class WatchFolderService { folderPath, uploadStrategy, ); - this.getAndSyncDiffOfFiles(); + this.syncWithDisk(); } catch (e) { log.error("error while adding watch mapping", e); } @@ -576,7 +577,7 @@ class WatchFolderService { resumePausedSync() { this.isPaused = false; - this.getAndSyncDiffOfFiles(); + this.syncWithDisk(); } } @@ -584,12 +585,6 @@ const watchFolderService = new WatchFolderService(); export default watchFolderService; -const getParentFolderName = (filePath: string) => { - const folderPath = filePath.substring(0, filePath.lastIndexOf("/")); - const folderName = folderPath.substring(folderPath.lastIndexOf("/") + 1); - return folderName; -}; - async function diskFileAddedCallback(file: ElectronFile) { const collectionNameAndFolderPath = await watchFolderService.getCollectionNameAndFolderPath(file.path); @@ -669,37 +664,35 @@ function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) { } /** - * Determine which events we need to process to synchronize the watched albums - * with the corresponding on disk folders. + * Determine which events we need to process to synchronize the watched on-disk + * folders to their corresponding collections. * - * Also return a list of previously created folder watches for this there is no - * longer any no corresponding folder on disk. + * Also return a list of previously created folder watches for which there is no + * longer any no corresponding directory on disk. */ -const syncWithDisk = async ( +const deduceEvents = async ( mappings: FolderWatch[], ): Promise<{ - events: EventQueueItem[]; - nonExistentFolderPaths: string[]; + events: WatchEvent[]; + deletedFolderPaths: string[]; }> => { const activeMappings = []; - const nonExistentFolderPaths: string[] = []; + const deletedFolderPaths: string[] = []; for (const mapping of mappings) { const valid = await electron.fs.isDir(mapping.folderPath); - if (!valid) nonExistentFolderPaths.push(mapping.folderPath); + if (!valid) deletedFolderPaths.push(mapping.folderPath); else activeMappings.push(mapping); } - const events: EventQueueItem[] = []; + const events: WatchEvent[] = []; for (const mapping of activeMappings) { const folderPath = mapping.folderPath; - const paths = (await electron.fs.listFiles(folderPath)) + const paths = (await electron.watch.findFiles(folderPath)) // Filter out hidden files (files whose names begins with a dot) - .filter((n) => !n.startsWith(".")) - // Prepend folderPath to get the full path - .map((f) => `${folderPath}/${f}`); + .filter((path) => !nameAndExtension(path)[0].startsWith(".")); // Files that are on disk but not yet synced. const pathsToUpload = paths.filter( @@ -708,7 +701,7 @@ const syncWithDisk = async ( for (const path of pathsToUpload) events.push({ - type: "upload", + action: "upload", collectionName: getCollectionNameForMapping(mapping, path), folderPath, filePath: path, @@ -728,7 +721,7 @@ const syncWithDisk = async ( }); } - return { events, nonExistentFolderPaths }; + return { events, deletedFolderPaths }; }; function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) { @@ -743,6 +736,16 @@ const getCollectionNameForMapping = ( filePath: string, ) => { return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER - ? getParentFolderName(filePath) + ? parentDirectoryName(filePath) : mapping.rootFolderName; }; + +const parentDirectoryName = (filePath: string) => { + const components = filePath.split("/"); + const parentName = components[components.length - 2]; + if (!parentName) + throw new Error( + `Unexpected file path without a parent folder: ${filePath}`, + ); + return parentName; +}; diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index b14c8e2704..5650d98748 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -205,18 +205,6 @@ export interface Electron { * directory. */ isDir: (dirPath: string) => Promise; - - /** - * Return a list of the file names of the files in the given directory. - * - * Note: - * - * - This is not recursive, it will only return the names of direct - * children. - * - * - It will return only the names of files, not directories. - */ - listFiles: (dirPath: string) => Promise; }; /* @@ -303,15 +291,32 @@ export interface Electron { // - Watch /** - * Get the latest state of the watched folders. + * Functions tailored for the folder watch functionality * - * We persist the folder watches that the user has setup. This function goes - * through that list, prunes any folders that don't exist on disk anymore, - * and for each, also returns a list of files that exist in that folder. + * [Note: Folder vs Directory in the context of FolderWatch-es] + * + * A note on terminology: The word "folder" is used to the top level root + * folder for which a {@link FolderWatch} has been added. This folder is + * also in 1-1 correspondence to be a directory on the user's disk. It can + * have other, nested directories too (which may or may not be getting + * mapped to separate Ente collections), but we'll not refer to these nested + * directories as folders - only the root of the tree, which the user + * dragged/dropped or selected to set up the folder watch, will be referred + * to as a folder when naming things. */ - folderWatchesAndFilesTherein: () => Promise< - [watch: FolderWatch, files: ElectronFile[]][] - >; + watch: { + /** + * Return the paths of all the files under the given folder. + * + * This function walks the directory tree starting at {@link folderPath} + * and returns a list of the absolute paths of all the files that exist + * therein. It will recursively traverse into nested directories, and + * return the absolute paths of the files there too. + * + * The returned paths are guaranteed to use POSIX separators ('/'). + */ + findFiles: (folderPath: string) => Promise; + }; registerWatcherFunctions: ( addFile: (file: ElectronFile) => Promise, @@ -327,6 +332,15 @@ export interface Electron { removeWatchMapping: (folderPath: string) => Promise; + /** + * TODO(MR): Outdated description + * Get the latest state of the watched folders. + * + * We persist the folder watches that the user has setup. This function goes + * through that list, prunes any folders that don't exist on disk anymore, + * and for each, also returns a list of files that exist in that folder. + */ + getWatchMappings: () => Promise; updateWatchMappingSyncedFiles: (