TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[desktop] Reconcile exported files with disk on app start and resync

Browse Source
This commit is contained in:
Manav Rathi
2024-06-28 13:35:34 +05:30
parent 22e7bcb3b0
commit 22fc61633b
4 changed files with 126 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
desktop/CHANGELOG.md
View File

@@ -7,6 +7,7 @@
- Add a option to set and use a custom endpoint.
- Fix an issue preventing subscription purchases and renewals.
- Clear cached password after changing it on a different device.
- Reconcile exported files with disk on app start and resync.
- .
## v1.7.1

9
web/apps/photos/src/components/ExportFinished.tsx
View File

@@ -19,7 +19,8 @@ interface Props {
collectionNameMap: Map<number, string>;
onHide: () => void;
lastExportTime: number;
startExport: () => void;
/** Called when the user presses the "Resync" button. */
onResync: () => void;
}
export default function ExportFinished(props: Props) {
@@ -67,11 +68,7 @@ export default function ExportFinished(props: Props) {
<Button color="secondary" size="large" onClick={props.onHide}>
{t("CLOSE")}
</Button>
<Button
size="large"
color="primary"
onClick={props.startExport}
>
<Button size="large" color="primary" onClick={props.onResync}>
{t("EXPORT_AGAIN")}
</Button>
</DialogActions>

9
web/apps/photos/src/components/ExportModal.tsx
View File

@@ -23,6 +23,7 @@ import { useContext, useEffect, useState } from "react";
import exportService, {
ExportStage,
selectAndPrepareExportDirectory,
type ExportOpts,
} from "services/export";
import { ExportProgress, ExportSettings } from "types/export";
import { getExportDirectoryDoesNotExistMessage } from "utils/ui";
@@ -144,10 +145,10 @@ export default function ExportModal(props: Props) {
setContinuousExport(newContinuousExport);
};
const startExport = async () => {
const startExport = async (opts?: ExportOpts) => {
if (!(await verifyExportFolderExists())) return;
await exportService.scheduleExport();
await exportService.scheduleExport(opts ?? {});
};
const stopExport = () => {
@@ -240,7 +241,7 @@ const ExportDynamicContent = ({
collectionNameMap,
}: {
exportStage: ExportStage;
startExport: () => void;
startExport: (opts?: ExportOpts) => void;
stopExport: () => void;
onHide: () => void;
lastExportTime: number;
@@ -273,7 +274,7 @@ const ExportDynamicContent = ({
lastExportTime={lastExportTime}
pendingExports={pendingExports}
collectionNameMap={collectionNameMap}
startExport={startExport}
onResync={() => startExport({ resync: true })}
/>
);

127
web/apps/photos/src/services/export/index.ts
View File

@@ -80,9 +80,22 @@ export const NULL_EXPORT_RECORD: ExportRecord = {
collectionExportNames: {},
};
export interface ExportOpts {
/**
* If true, perform an additional on-disk check to determine which files
* need to be exported.
*
* This has performance implications for huge libraries, so we only do this:
* - For the first export after an app start
* - If the user explicitly presses the "Resync" button.
*/
resync?: boolean;
}
class ExportService {
private exportSettings: ExportSettings;
private exportInProgress: RequestCanceller = null;
private resync = true;
private reRunNeeded = false;
private exportRecordUpdater = new QueueProcessor<ExportRecord>();
private fileReader: FileReader = null;
@@ -164,6 +177,16 @@ class ExportService {
this.uiUpdater.setLastExportTime(exportTime);
}
private resyncOnce() {
const resync = this.resync;
this.resync = false;
return resync;
}
resumeExport() {
this.scheduleExport({ resync: this.resyncOnce() });
}
enableContinuousExport() {
try {
if (this.continuousExportEventHandler) {
@@ -172,7 +195,7 @@ class ExportService {
}
log.info("enabling continuous export");
this.continuousExportEventHandler = () => {
this.scheduleExport();
this.scheduleExport({ resync: this.resyncOnce() });
};
this.continuousExportEventHandler();
eventBus.addListener(
@@ -225,6 +248,7 @@ class ExportService {
const unExportedFiles = getUnExportedFiles(
userPersonalFiles,
exportRecord,
undefined,
);
return unExportedFiles;
} catch (e) {
@@ -276,7 +300,7 @@ class ExportService {
}
}
scheduleExport = async () => {
scheduleExport = async (exportOpts: ExportOpts) => {
try {
if (this.exportInProgress) {
log.info("export in progress, scheduling re-run");
@@ -297,7 +321,7 @@ class ExportService {
const exportFolder = this.getExportSettings()?.folder;
await this.preExport(exportFolder);
log.info("export started");
await this.runExport(exportFolder, isCanceled);
await this.runExport(exportFolder, isCanceled, exportOpts);
log.info("export completed");
} finally {
if (isCanceled.status) {
@@ -312,7 +336,7 @@ class ExportService {
if (this.reRunNeeded) {
this.reRunNeeded = false;
log.info("re-running export");
setTimeout(() => this.scheduleExport(), 0);
setTimeout(() => this.scheduleExport(exportOpts), 0);
}
}
}
@@ -329,6 +353,7 @@ class ExportService {
private async runExport(
exportFolder: string,
isCanceled: CancellationStatus,
{ resync }: ExportOpts,
) {
try {
const user: User = getData(LS_KEYS.USER);
@@ -370,10 +395,23 @@ class ExportService {
personalFiles,
exportRecord,
);
const diskFileRecordIDs = resync
? await readOnDiskFileExportRecordIDs(
personalFiles,
collectionIDExportNameMap,
exportFolder,
exportRecord,
isCanceled,
)
: undefined;
const filesToExport = getUnExportedFiles(
personalFiles,
exportRecord,
diskFileRecordIDs,
);
const deletedExportedCollections = getDeletedExportedCollections(
nonEmptyPersonalCollections,
exportRecord,
@@ -1122,7 +1160,7 @@ export const resumeExportsIfNeeded = async () => {
}
if (isExportInProgress(exportRecord.stage)) {
log.debug(() => "Resuming in-progress export");
exportService.scheduleExport();
exportService.resumeExport();
}
};
@@ -1229,21 +1267,90 @@ const getDeletedExportedCollections = (
return deletedExportedCollections;
};
/**
* Return export record IDs of {@link files} for which there is also exists a
* file on disk.
*/
const readOnDiskFileExportRecordIDs = async (
files: EnteFile[],
collectionIDFolderNameMap: Map<number, string>,
exportDir: string,
exportRecord: ExportRecord,
isCanceled: CancellationStatus,
): Promise<Set<string>> => {
const fs = ensureElectron().fs;
const result = new Set<string>();
if (!(await fs.exists(exportDir))) return result;
const fileExportNames = exportRecord.fileExportNames ?? {};
for (const file of files) {
if (isCanceled.status) throw Error(CustomError.EXPORT_STOPPED);
const collectionExportName = collectionIDFolderNameMap.get(
file.collectionID,
);
if (!collectionExportName) continue;
const collectionExportPath = `${exportDir}/${collectionExportName}`;
const recordID = getExportRecordFileUID(file);
const exportName = fileExportNames[recordID];
if (!exportName) continue;
let fileName: string;
let fileName2: string | undefined; // Live photos have 2 parts
if (isLivePhotoExportName(exportName)) {
const { image, video } = parseLivePhotoExportName(exportName);
fileName = image;
fileName2 = video;
} else {
fileName = exportName;
}
const filePath = `${collectionExportPath}/${fileName}`;
if (await fs.exists(filePath)) {
// Also check that the sibling part exists (if any).
if (fileName2) {
const filePath2 = `${collectionExportPath}/${fileName2}`;
if (await fs.exists(filePath2)) result.add(recordID);
} else {
result.add(recordID);
}
}
}
return result;
};
/**
* Return the list of files from amongst {@link allFiles} that still need to be
* exported.
*
* @param allFiles The list of files to export.
*
* @param exportRecord The export record containing bookeeping for the export.
*
* @paramd diskFileRecordIDs (Optional) The export record IDs of files from
* amongst {@link allFiles} that already exist on disk. If provided (e.g. when
* doing a resync), we perform an extra check for on-disk existence instead of
* relying solely on the export record.
*/
const getUnExportedFiles = (
allFiles: EnteFile[],
exportRecord: ExportRecord,
diskFileRecordIDs: Set<string> | undefined,
) => {
if (!exportRecord?.fileExportNames) {
return allFiles;
}
const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames));
const unExportedFiles = allFiles.filter((file) => {
if (!exportedFiles.has(getExportRecordFileUID(file))) {
return true;
}
return allFiles.filter((file) => {
const recordID = getExportRecordFileUID(file);
if (!exportedFiles.has(recordID)) return true;
if (diskFileRecordIDs && !diskFileRecordIDs.has(recordID)) return true;
return false;
});
return unExportedFiles;
};
const getDeletedExportedFiles = (