TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP 2

Browse Source
This commit is contained in:
Manav Rathi
2024-04-17 17:00:51 +05:30
parent 52c35108ca
commit a22423d039
5 changed files with 95 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
desktop/src/main/ipc.ts
View File

@@ -20,7 +20,6 @@ import {
import {
fsExists,
fsIsDir,
fsListFiles,
fsMkdirIfNeeded,
fsReadTextFile,
fsRename,
@@ -56,6 +55,7 @@ import {
} from "./services/upload";
import {
addWatchMapping,
findFiles,
getWatchMappings,
removeWatchMapping,
updateWatchMappingIgnoredFiles,
@@ -135,8 +135,6 @@ export const attachIPCHandlers = () => {
ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath));
// - Conversion
ipcMain.handle("convertToJPEG", (_, fileData, filename) =>
@@ -219,6 +217,10 @@ export const attachIPCHandlers = () => {
export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => {
// - Watch
ipcMain.handle("findFiles", (_, folderPath: string) =>
findFiles(folderPath),
);
ipcMain.handle(
"addWatchMapping",
(

24
desktop/src/main/services/watch.ts
View File

@@ -1,33 +1,23 @@
import type { FSWatcher } from "chokidar";
import ElectronLog from "electron-log";
import fs from "node:fs/promises";
import path from "node:path";
import { FolderWatch, WatchStoreType } from "../../types/ipc";
import { watchStore } from "../stores/watch.store";
import path from "node:path";
import fs from "node:fs/promises";
/**
* Return the paths of all the files under the given directory (recursive).
*
* This function walks the directory tree starting at {@link dirPath}, and
* returns a list of the absolute paths of all the files that exist therein. It
* will recursively traverse into nested directories, and return the absolute
* paths of the files there too.
*
* The returned paths are guaranteed to use POSIX separators ('/').
*/
export const findFiles = async (dirPath: string) => {
const items = await fs.readdir(dirPath, { withFileTypes: true })
const items = await fs.readdir(dirPath, { withFileTypes: true });
let paths: string[] = [];
for (const item of items) {
const itemPath = path.posix.join(dirPath, item.name);
if (item.isFile()) {
paths.push(itemPath)
paths.push(itemPath);
} else if (item.isDirectory()) {
paths = [...paths, ...await findFiles(itemPath)]
paths = [...paths, ...(await findFiles(itemPath))];
}
}
return paths
}
return paths;
};
export const addWatchMapping = async (
watcher: FSWatcher,

10
desktop/src/preload.ts
View File

@@ -121,9 +121,6 @@ const fsWriteFile = (path: string, contents: string): Promise<void> =>
const fsIsDir = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("fsIsDir", dirPath);
const fsListFiles = (dirPath: string): Promise<string[]> =>
ipcRenderer.invoke("fsListFiles", dirPath);
// - AUDIT below this
// - Conversion
@@ -194,6 +191,9 @@ const showUploadZipDialog = (): Promise<{
// - Watch
const findFiles = (folderPath: string): Promise<string[]> =>
ipcRenderer.invoke("findFiles", folderPath);
const registerWatcherFunctions = (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
@@ -325,7 +325,6 @@ contextBridge.exposeInMainWorld("electron", {
readTextFile: fsReadTextFile,
writeFile: fsWriteFile,
isDir: fsIsDir,
listFiles: fsListFiles,
},
// - Conversion
@@ -346,6 +345,9 @@ contextBridge.exposeInMainWorld("electron", {
showUploadZipDialog,
// - Watch
watch: {
findFiles,
},
registerWatcherFunctions,
addWatchMapping,
removeWatchMapping,

85
web/apps/photos/src/services/watch.ts
View File

@@ -4,6 +4,7 @@
*/
import { ensureElectron } from "@/next/electron";
import { nameAndExtension } from "@/next/file";
import log from "@/next/log";
import type { FolderWatch } from "@/next/types/ipc";
import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload";
@@ -19,15 +20,15 @@ import { removeFromCollection } from "./collectionService";
import { getLocalFiles } from "./fileService";
/**
* A file system event encapsulates a change that has occurred on disk that
* needs us to take some action within Ente to synchronize with the user's
* (Ente) albums.
* A file system watch event encapsulates a change that has occurred on disk
* that needs us to take some action within Ente to synchronize with the user's
* Ente collections.
*
* Events get added in two ways:
*
* - When the app starts, it reads the current state of files on disk and
* compares that with its last known state to determine what all events it
* missed. This is easier than it sounds as we have only two events, add and
* missed. This is easier than it sounds as we have only two events: add and
* remove.
*
* - When the app is running, it gets live notifications from our file system
@@ -35,20 +36,20 @@ import { getLocalFiles } from "./fileService";
* which the app then enqueues onto the event queue if they pertain to the
* files we're interested in.
*/
interface FSEvent {
interface WatchEvent {
/** The action to take */
action: "upload" | "trash";
/** The path of the root folder corresponding to the {@link FolderWatch}. */
folderPath: string;
/** If applicable, the name of the (Ente) collection the file belongs to. */
/** The name of the Ente collection the file belongs to. */
collectionName?: string;
/** The absolute path to the file under consideration. */
filePath: string;
}
class WatchFolderService {
private eventQueue: FSEvent[] = [];
private currentEvent: FSEvent;
private eventQueue: WatchEvent[] = [];
private currentEvent: WatchEvent;
private currentlySyncedMapping: WatchMapping;
private trashingDirQueue: string[] = [];
private isEventRunning: boolean = false;
@@ -87,26 +88,26 @@ class WatchFolderService {
this.setWatchFolderServiceIsRunning =
setWatchFolderServiceIsRunning;
this.setupWatcherFunctions();
await this.getAndSyncDiffOfFiles();
await this.syncWithDisk();
} catch (e) {
log.error("error while initializing watch service", e);
}
}
async getAndSyncDiffOfFiles() {
private async syncWithDisk() {
try {
const electron = ensureElectron();
const mappings = await electron.getWatchMappings();
if (!mappings) return;
this.eventQueue = [];
const { events, nonExistentFolderPaths } =
await syncWithDisk(mappings);
const { events, deletedFolderPaths } = await deduceEvents(mappings);
this.eventQueue = [...this.eventQueue, ...events];
this.debouncedRunNextEvent();
for (const path of nonExistentFolderPaths)
for (const path of deletedFolderPaths)
electron.removeWatchMapping(path);
this.debouncedRunNextEvent();
} catch (e) {
log.error("Ignoring error while syncing watched folders", e);
}
@@ -116,9 +117,9 @@ class WatchFolderService {
return this.currentEvent?.folderPath === mapping.folderPath;
}
pushEvent(event: EventQueueItem) {
private pushEvent(event: WatchEvent) {
this.eventQueue.push(event);
log.info("FS event", event);
log.info("Watch event", event);
this.debouncedRunNextEvent();
}
@@ -145,7 +146,7 @@ class WatchFolderService {
folderPath,
uploadStrategy,
);
this.getAndSyncDiffOfFiles();
this.syncWithDisk();
} catch (e) {
log.error("error while adding watch mapping", e);
}
@@ -576,7 +577,7 @@ class WatchFolderService {
resumePausedSync() {
this.isPaused = false;
this.getAndSyncDiffOfFiles();
this.syncWithDisk();
}
}
@@ -584,12 +585,6 @@ const watchFolderService = new WatchFolderService();
export default watchFolderService;
const getParentFolderName = (filePath: string) => {
const folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
const folderName = folderPath.substring(folderPath.lastIndexOf("/") + 1);
return folderName;
};
async function diskFileAddedCallback(file: ElectronFile) {
const collectionNameAndFolderPath =
await watchFolderService.getCollectionNameAndFolderPath(file.path);
@@ -669,37 +664,35 @@ function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) {
}
/**
* Determine which events we need to process to synchronize the watched albums
* with the corresponding on disk folders.
* Determine which events we need to process to synchronize the watched on-disk
* folders to their corresponding collections.
*
* Also return a list of previously created folder watches for this there is no
* longer any no corresponding folder on disk.
* Also return a list of previously created folder watches for which there is no
* longer any no corresponding directory on disk.
*/
const syncWithDisk = async (
const deduceEvents = async (
mappings: FolderWatch[],
): Promise<{
events: EventQueueItem[];
nonExistentFolderPaths: string[];
events: WatchEvent[];
deletedFolderPaths: string[];
}> => {
const activeMappings = [];
const nonExistentFolderPaths: string[] = [];
const deletedFolderPaths: string[] = [];
for (const mapping of mappings) {
const valid = await electron.fs.isDir(mapping.folderPath);
if (!valid) nonExistentFolderPaths.push(mapping.folderPath);
if (!valid) deletedFolderPaths.push(mapping.folderPath);
else activeMappings.push(mapping);
}
const events: EventQueueItem[] = [];
const events: WatchEvent[] = [];
for (const mapping of activeMappings) {
const folderPath = mapping.folderPath;
const paths = (await electron.fs.listFiles(folderPath))
const paths = (await electron.watch.findFiles(folderPath))
// Filter out hidden files (files whose names begins with a dot)
.filter((n) => !n.startsWith("."))
// Prepend folderPath to get the full path
.map((f) => `${folderPath}/${f}`);
.filter((path) => !nameAndExtension(path)[0].startsWith("."));
// Files that are on disk but not yet synced.
const pathsToUpload = paths.filter(
@@ -708,7 +701,7 @@ const syncWithDisk = async (
for (const path of pathsToUpload)
events.push({
type: "upload",
action: "upload",
collectionName: getCollectionNameForMapping(mapping, path),
folderPath,
filePath: path,
@@ -728,7 +721,7 @@ const syncWithDisk = async (
});
}
return { events, nonExistentFolderPaths };
return { events, deletedFolderPaths };
};
function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) {
@@ -743,6 +736,16 @@ const getCollectionNameForMapping = (
filePath: string,
) => {
return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER
? getParentFolderName(filePath)
? parentDirectoryName(filePath)
: mapping.rootFolderName;
};
const parentDirectoryName = (filePath: string) => {
const components = filePath.split("/");
const parentName = components[components.length - 2];
if (!parentName)
throw new Error(
`Unexpected file path without a parent folder: ${filePath}`,
);
return parentName;
};

52
web/packages/next/types/ipc.ts
View File

@@ -205,18 +205,6 @@ export interface Electron {
* directory.
*/
isDir: (dirPath: string) => Promise<boolean>;
/**
* Return a list of the file names of the files in the given directory.
*
* Note:
*
* - This is not recursive, it will only return the names of direct
* children.
*
* - It will return only the names of files, not directories.
*/
listFiles: (dirPath: string) => Promise<string[]>;
};
/*
@@ -303,15 +291,32 @@ export interface Electron {
// - Watch
/**
* Get the latest state of the watched folders.
* Functions tailored for the folder watch functionality
*
* We persist the folder watches that the user has setup. This function goes
* through that list, prunes any folders that don't exist on disk anymore,
* and for each, also returns a list of files that exist in that folder.
* [Note: Folder vs Directory in the context of FolderWatch-es]
*
* A note on terminology: The word "folder" is used to the top level root
* folder for which a {@link FolderWatch} has been added. This folder is
* also in 1-1 correspondence to be a directory on the user's disk. It can
* have other, nested directories too (which may or may not be getting
* mapped to separate Ente collections), but we'll not refer to these nested
* directories as folders - only the root of the tree, which the user
* dragged/dropped or selected to set up the folder watch, will be referred
* to as a folder when naming things.
*/
folderWatchesAndFilesTherein: () => Promise<
[watch: FolderWatch, files: ElectronFile[]][]
>;
watch: {
/**
* Return the paths of all the files under the given folder.
*
* This function walks the directory tree starting at {@link folderPath}
* and returns a list of the absolute paths of all the files that exist
* therein. It will recursively traverse into nested directories, and
* return the absolute paths of the files there too.
*
* The returned paths are guaranteed to use POSIX separators ('/').
*/
findFiles: (folderPath: string) => Promise<string[]>;
};
registerWatcherFunctions: (
addFile: (file: ElectronFile) => Promise<void>,
@@ -327,6 +332,15 @@ export interface Electron {
removeWatchMapping: (folderPath: string) => Promise<void>;
/**
* TODO(MR): Outdated description
* Get the latest state of the watched folders.
*
* We persist the folder watches that the user has setup. This function goes
* through that list, prunes any folders that don't exist on disk anymore,
* and for each, also returns a list of files that exist in that folder.
*/
getWatchMappings: () => Promise<FolderWatch[]>;
updateWatchMappingSyncedFiles: (