From f7e961e86b310a1b2697f6cf87ff220c9b52c519 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 16:44:55 +0530 Subject: [PATCH 01/10] Inline --- web/apps/photos/src/constants/export.ts | 1 - .../photos/src/services/export/migration.ts | 143 +++++++++++++++-- web/apps/photos/src/utils/export/index.ts | 14 +- web/apps/photos/src/utils/export/migration.ts | 146 ------------------ 4 files changed, 136 insertions(+), 168 deletions(-) delete mode 100644 web/apps/photos/src/utils/export/migration.ts diff --git a/web/apps/photos/src/constants/export.ts b/web/apps/photos/src/constants/export.ts index cd6c0c0ee7..87be9f2384 100644 --- a/web/apps/photos/src/constants/export.ts +++ b/web/apps/photos/src/constants/export.ts @@ -1,6 +1,5 @@ export const ENTE_METADATA_FOLDER = "metadata"; -export const ENTE_TRASH_FOLDER = "Trash"; export enum ExportStage { INIT = 0, diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 6c79420eda..69a6974027 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -2,6 +2,7 @@ import log from "@/next/log"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { sleep } from "@ente/shared/utils"; +import { ENTE_METADATA_FOLDER } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import downloadManager from "services/download"; @@ -15,6 +16,7 @@ import { ExportRecordV0, ExportRecordV1, ExportRecordV2, + ExportedCollectionPaths, FileExportNames, } from "types/export"; import { EnteFile } from "types/file"; @@ -25,19 +27,9 @@ import { getExportRecordFileUID, getLivePhotoExportName, getMetadataFolderExportPath, + sanitizeName, } from "utils/export"; -import { - convertCollectionIDFolderPathObjectToMap, - getExportedFiles, - getFileMetadataSavePath, - getFileSavePath, - getOldCollectionFolderPath, - getOldFileMetadataSavePath, - getOldFileSavePath, - getUniqueCollectionFolderPath, - getUniqueFileExportNameForMigration, - getUniqueFileSaveName, -} from "utils/export/migration"; +import { splitFilenameAndExtension } from "utils/ffmpeg"; import { getIDBasedSortedFiles, getPersonalFiles, @@ -475,3 +467,130 @@ async function removeCollectionExportMissingMetadataFolder( }; await exportService.updateExportRecord(exportDir, updatedExportRecord); } + +const convertCollectionIDFolderPathObjectToMap = ( + exportedCollectionPaths: ExportedCollectionPaths, +): Map => { + return new Map( + Object.entries(exportedCollectionPaths ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }), + ); +}; + +const getExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2, +) => { + if (!exportRecord?.exportedFiles) { + return []; + } + const exportedFileIds = new Set(exportRecord?.exportedFiles); + const exportedFiles = allFiles.filter((file) => { + if (exportedFileIds.has(getExportRecordFileUID(file))) { + return true; + } else { + return false; + } + }); + return exportedFiles; +}; + +const oldSanitizeName = (name: string) => + name.replaceAll("/", "_").replaceAll(" ", "_"); + +const getUniqueCollectionFolderPath = async ( + dir: string, + collectionName: string, +): Promise => { + let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; + let count = 1; + while (await exportService.exists(collectionFolderPath)) { + collectionFolderPath = `${dir}/${sanitizeName( + collectionName, + )}(${count})`; + count++; + } + return collectionFolderPath; +}; + +export const getMetadataFolderPath = (collectionFolderPath: string) => + `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`; + +const getUniqueFileSaveName = async ( + collectionPath: string, + filename: string, +) => { + let fileSaveName = sanitizeName(filename); + let count = 1; + while ( + await exportService.exists( + getFileSavePath(collectionPath, fileSaveName), + ) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileSaveName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileSaveName; +}; + +const getFileMetadataSavePath = ( + collectionFolderPath: string, + fileSaveName: string, +) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`; + +const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) => + `${collectionFolderPath}/${fileSaveName}`; + +const getOldCollectionFolderPath = ( + dir: string, + collectionID: number, + collectionName: string, +) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`; + +const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) => + `${collectionFolderPath}/${file.id}_${oldSanitizeName( + file.metadata.title, + )}`; + +const getOldFileMetadataSavePath = ( + collectionFolderPath: string, + file: EnteFile, +) => + `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${ + file.id + }_${oldSanitizeName(file.metadata.title)}.json`; + +const getUniqueFileExportNameForMigration = ( + collectionPath: string, + filename: string, + usedFilePaths: Map>, +) => { + let fileExportName = sanitizeName(filename); + let count = 1; + while ( + usedFilePaths + .get(collectionPath) + ?.has(getFileSavePath(collectionPath, fileExportName)) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileExportName = `${filenameParts[0]}(${count})`; + } + count++; + } + if (!usedFilePaths.has(collectionPath)) { + usedFilePaths.set(collectionPath, new Set()); + } + usedFilePaths + .get(collectionPath) + .add(getFileSavePath(collectionPath, fileExportName)); + return fileExportName; +}; diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts index a98e431b24..278b6cde31 100644 --- a/web/apps/photos/src/utils/export/index.ts +++ b/web/apps/photos/src/utils/export/index.ts @@ -1,3 +1,6 @@ +import { formatDateTimeShort } from "@ente/shared/time/format"; +import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; +import sanitize from "sanitize-filename"; import exportService from "services/export"; import { Collection } from "types/collection"; import { @@ -5,20 +8,13 @@ import { ExportRecord, FileExportNames, } from "types/export"; - import { EnteFile } from "types/file"; - -import { formatDateTimeShort } from "@ente/shared/time/format"; -import { - ENTE_METADATA_FOLDER, - ENTE_TRASH_FOLDER, - ExportStage, -} from "constants/export"; -import sanitize from "sanitize-filename"; import { Metadata } from "types/upload"; import { getCollectionUserFacingName } from "utils/collection"; import { splitFilenameAndExtension } from "utils/file"; +export const ENTE_TRASH_FOLDER = "Trash"; + export const getExportRecordFileUID = (file: EnteFile) => `${file.id}_${file.collectionID}_${file.updationTime}`; diff --git a/web/apps/photos/src/utils/export/migration.ts b/web/apps/photos/src/utils/export/migration.ts deleted file mode 100644 index c8988cac41..0000000000 --- a/web/apps/photos/src/utils/export/migration.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { ENTE_METADATA_FOLDER } from "constants/export"; -import exportService from "services/export"; -import { - ExportedCollectionPaths, - ExportRecordV0, - ExportRecordV1, - ExportRecordV2, -} from "types/export"; -import { EnteFile } from "types/file"; -import { splitFilenameAndExtension } from "utils/ffmpeg"; -import { getExportRecordFileUID, sanitizeName } from "."; - -export const convertCollectionIDFolderPathObjectToMap = ( - exportedCollectionPaths: ExportedCollectionPaths, -): Map => { - return new Map( - Object.entries(exportedCollectionPaths ?? {}).map((e) => { - return [Number(e[0]), String(e[1])]; - }), - ); -}; - -export const getExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2, -) => { - if (!exportRecord?.exportedFiles) { - return []; - } - const exportedFileIds = new Set(exportRecord?.exportedFiles); - const exportedFiles = allFiles.filter((file) => { - if (exportedFileIds.has(getExportRecordFileUID(file))) { - return true; - } else { - return false; - } - }); - return exportedFiles; -}; - -export const oldSanitizeName = (name: string) => - name.replaceAll("/", "_").replaceAll(" ", "_"); - -export const getUniqueCollectionFolderPath = async ( - dir: string, - collectionName: string, -): Promise => { - let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; - let count = 1; - while (await exportService.exists(collectionFolderPath)) { - collectionFolderPath = `${dir}/${sanitizeName( - collectionName, - )}(${count})`; - count++; - } - return collectionFolderPath; -}; - -export const getMetadataFolderPath = (collectionFolderPath: string) => - `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`; - -export const getUniqueFileSaveName = async ( - collectionPath: string, - filename: string, -) => { - let fileSaveName = sanitizeName(filename); - let count = 1; - while ( - await exportService.exists( - getFileSavePath(collectionPath, fileSaveName), - ) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileSaveName = `${filenameParts[0]}(${count})`; - } - count++; - } - return fileSaveName; -}; - -export const getOldFileSaveName = (filename: string, fileID: number) => - `${fileID}_${oldSanitizeName(filename)}`; - -export const getFileMetadataSavePath = ( - collectionFolderPath: string, - fileSaveName: string, -) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`; - -export const getFileSavePath = ( - collectionFolderPath: string, - fileSaveName: string, -) => `${collectionFolderPath}/${fileSaveName}`; - -export const getOldCollectionFolderPath = ( - dir: string, - collectionID: number, - collectionName: string, -) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`; - -export const getOldFileSavePath = ( - collectionFolderPath: string, - file: EnteFile, -) => - `${collectionFolderPath}/${file.id}_${oldSanitizeName( - file.metadata.title, - )}`; - -export const getOldFileMetadataSavePath = ( - collectionFolderPath: string, - file: EnteFile, -) => - `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${ - file.id - }_${oldSanitizeName(file.metadata.title)}.json`; - -export const getUniqueFileExportNameForMigration = ( - collectionPath: string, - filename: string, - usedFilePaths: Map>, -) => { - let fileExportName = sanitizeName(filename); - let count = 1; - while ( - usedFilePaths - .get(collectionPath) - ?.has(getFileSavePath(collectionPath, fileExportName)) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileExportName = `${filenameParts[0]}(${count})`; - } - count++; - } - if (!usedFilePaths.has(collectionPath)) { - usedFilePaths.set(collectionPath, new Set()); - } - usedFilePaths - .get(collectionPath) - .add(getFileSavePath(collectionPath, fileExportName)); - return fileExportName; -}; From ca114a467dd1354384722e38ac4e5543cd49b8c9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 16:50:12 +0530 Subject: [PATCH 02/10] Inline 1 --- web/apps/photos/src/services/export/index.ts | 6 +++++- web/apps/photos/src/services/export/migration.ts | 3 +-- web/apps/photos/src/utils/collection/index.ts | 10 ++-------- web/apps/photos/src/utils/export/index.ts | 9 +-------- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index f2e90139a1..0d6d5ac32a 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -27,7 +27,6 @@ import { import { convertCollectionIDExportNameObjectToMap, convertFileIDExportNameObjectToMap, - getCollectionExportPath, getCollectionExportedFiles, getCollectionIDFromFileUID, getDeletedExportedCollections, @@ -1219,3 +1218,8 @@ class ExportService { }; } export default new ExportService(); + +const getCollectionExportPath = ( + exportFolder: string, + collectionExportName: string, +) => `${exportFolder}/${collectionExportName}`; diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 69a6974027..812eafad4c 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -22,7 +22,6 @@ import { import { EnteFile } from "types/file"; import { getNonEmptyPersonalCollections } from "utils/collection"; import { - getCollectionExportPath, getCollectionIDFromFileUID, getExportRecordFileUID, getLivePhotoExportName, @@ -433,7 +432,7 @@ async function removeCollectionExportMissingMetadataFolder( if ( await exportService.exists( getMetadataFolderExportPath( - getCollectionExportPath(exportDir, collectionExportName), + `${exportDir}/${collectionExportName}`, ), ) ) { diff --git a/web/apps/photos/src/utils/collection/index.ts b/web/apps/photos/src/utils/collection/index.ts index c188615157..3b0f955e09 100644 --- a/web/apps/photos/src/utils/collection/index.ts +++ b/web/apps/photos/src/utils/collection/index.ts @@ -42,10 +42,7 @@ import { import { EnteFile } from "types/file"; import { SetFilesDownloadProgressAttributes } from "types/gallery"; import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata"; -import { - getCollectionExportPath, - getUniqueCollectionExportName, -} from "utils/export"; +import { getUniqueCollectionExportName } from "utils/export"; import { downloadFilesWithProgress } from "utils/file"; import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata"; @@ -176,10 +173,7 @@ async function createCollectionDownloadFolder( downloadDirPath, collectionName, ); - const collectionDownloadPath = getCollectionExportPath( - downloadDirPath, - collectionDownloadName, - ); + const collectionDownloadPath = `${downloadDirPath}/${collectionDownloadName}`; await exportService.checkExistsAndCreateDir(collectionDownloadPath); return collectionDownloadPath; } diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts index 278b6cde31..bedd7cc59a 100644 --- a/web/apps/photos/src/utils/export/index.ts +++ b/web/apps/photos/src/utils/export/index.ts @@ -200,9 +200,7 @@ export const getUniqueCollectionExportName = async ( let collectionExportName = sanitizeName(collectionName); let count = 1; while ( - (await exportService.exists( - getCollectionExportPath(dir, collectionExportName), - )) || + (await exportService.exists(`${dir}/${collectionExportName}`)) || collectionExportName === ENTE_TRASH_FOLDER ) { collectionExportName = `${sanitizeName(collectionName)}(${count})`; @@ -241,11 +239,6 @@ export const getFileMetadataExportPath = ( fileExportName: string, ) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; -export const getCollectionExportPath = ( - exportFolder: string, - collectionExportName: string, -) => `${exportFolder}/${collectionExportName}`; - export const getFileExportPath = ( collectionExportPath: string, fileExportName: string, From 284d7920b306cfaed204ca8701a2270f0fdc6c07 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 16:54:26 +0530 Subject: [PATCH 03/10] Inline 2 --- web/apps/photos/src/services/export/index.ts | 41 +++++--------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 0d6d5ac32a..67692a961d 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -483,11 +483,7 @@ class ExportService { await this.verifyExportFolderExists(exportFolder); const oldCollectionExportName = collectionIDExportNameMap.get(collection.id); - const oldCollectionExportPath = getCollectionExportPath( - exportFolder, - oldCollectionExportName, - ); - + const oldCollectionExportPath = `${exportFolder}/${oldCollectionExportName}`; const newCollectionExportName = await getUniqueCollectionExportName( exportFolder, @@ -496,11 +492,7 @@ class ExportService { log.info( `renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`, ); - const newCollectionExportPath = getCollectionExportPath( - exportFolder, - newCollectionExportName, - ); - + const newCollectionExportPath = `${exportFolder}/${newCollectionExportName}`; await this.addCollectionExportedRecord( exportFolder, collection.id, @@ -586,10 +578,7 @@ class ExportService { "collection is not empty, can't remove", ); } - const collectionExportPath = getCollectionExportPath( - exportFolder, - collectionExportName, - ); + const collectionExportPath = `${exportFolder}/${collectionExportName}`; await this.removeCollectionExportedRecord( exportFolder, collectionID, @@ -681,10 +670,7 @@ class ExportService { collectionExportName, ); } - const collectionExportPath = getCollectionExportPath( - exportDir, - collectionExportName, - ); + const collectionExportPath = `${exportDir}/${collectionExportName}`; await ensureElectron().checkExistsAndCreateDir( collectionExportPath, ); @@ -749,10 +735,10 @@ class ExportService { try { const fileExportName = fileIDExportNameMap.get(fileUID); const collectionID = getCollectionIDFromFileUID(fileUID); - const collectionExportPath = getCollectionExportPath( - exportDir, - collectionIDExportNameMap.get(collectionID), - ); + const collectionExportName = + collectionIDExportNameMap.get(collectionID); + const collectionExportPath = `${exportDir}/${collectionExportName}`; + await this.removeFileExportedRecord(exportDir, fileUID); try { if (isLivePhotoExportName(fileExportName)) { @@ -1036,10 +1022,7 @@ class ExportService { exportFolder, collectionName, ); - const collectionExportPath = getCollectionExportPath( - exportFolder, - collectionExportName, - ); + const collectionExportPath = `${exportFolder}/${collectionExportName}`; await ensureElectron().checkExistsAndCreateDir(collectionExportPath); await ensureElectron().checkExistsAndCreateDir( getMetadataFolderExportPath(collectionExportPath), @@ -1217,9 +1200,5 @@ class ExportService { return exportRecord; }; } -export default new ExportService(); -const getCollectionExportPath = ( - exportFolder: string, - collectionExportName: string, -) => `${exportFolder}/${collectionExportName}`; +export default new ExportService(); From c90ba63aad470eac2e1381569a198820c751c9a8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 16:57:04 +0530 Subject: [PATCH 04/10] Inline 1 --- web/apps/photos/src/services/export/index.ts | 6 +++++- web/apps/photos/src/utils/export/index.ts | 9 +-------- web/apps/photos/src/utils/file/index.ts | 12 +++++------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 67692a961d..02753d2992 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -32,7 +32,6 @@ import { getDeletedExportedCollections, getDeletedExportedFiles, getExportRecordFileUID, - getFileExportPath, getFileMetadataExportPath, getGoogleLikeMetadataFile, getLivePhotoExportName, @@ -1202,3 +1201,8 @@ class ExportService { } export default new ExportService(); + +export const getFileExportPath = ( + collectionExportPath: string, + fileExportName: string, +) => `${collectionExportPath}/${fileExportName}`; diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts index bedd7cc59a..76f1e1bb03 100644 --- a/web/apps/photos/src/utils/export/index.ts +++ b/web/apps/photos/src/utils/export/index.ts @@ -219,9 +219,7 @@ export const getUniqueFileExportName = async ( let fileExportName = sanitizeName(filename); let count = 1; while ( - await exportService.exists( - getFileExportPath(collectionExportPath, fileExportName), - ) + await exportService.exists(`${collectionExportPath}/${fileExportName}`) ) { const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); if (filenameParts[1]) { @@ -239,11 +237,6 @@ export const getFileMetadataExportPath = ( fileExportName: string, ) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; -export const getFileExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${fileExportName}`; - export const getTrashedFileExportPath = async ( exportDir: string, path: string, diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 089c5f40d4..d615ef707e 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -51,7 +51,7 @@ import { } from "types/gallery"; import { VISIBILITY_STATE } from "types/magicMetadata"; import { FileTypeInfo } from "types/upload"; -import { getFileExportPath, getUniqueFileExportName } from "utils/export"; +import { getUniqueFileExportName } from "utils/export"; import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000; @@ -818,7 +818,7 @@ async function downloadFileDesktop( ); const imageStream = generateStreamFromArrayBuffer(livePhoto.image); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, imageExportName), + `${downloadPath}/${imageExportName}`, imageStream, ); try { @@ -828,13 +828,11 @@ async function downloadFileDesktop( ); const videoStream = generateStreamFromArrayBuffer(livePhoto.video); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, videoExportName), + `${downloadPath}/${videoExportName}`, videoStream, ); } catch (e) { - await electron.deleteFile( - getFileExportPath(downloadPath, imageExportName), - ); + await electron.deleteFile(`${downloadPath}/${imageExportName}`); throw e; } } else { @@ -843,7 +841,7 @@ async function downloadFileDesktop( file.metadata.title, ); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, fileExportName), + `${downloadPath}/${fileExportName}`, updatedFileStream, ); } From 07fa5cb83d15a0230dff92458323606e22c41c87 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 16:58:49 +0530 Subject: [PATCH 05/10] Inline 2 --- web/apps/photos/src/services/export/index.ts | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 02753d2992..f5e06bc93d 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -745,10 +745,7 @@ class ExportService { image: imageExportName, video: videoExportName, } = parseLivePhotoExportName(fileExportName); - const imageExportPath = getFileExportPath( - collectionExportPath, - imageExportName, - ); + const imageExportPath = `${collectionExportPath}/${imageExportName}`; log.info( `moving image file ${imageExportPath} to trash folder`, ); @@ -777,10 +774,7 @@ class ExportService { ); } - const videoExportPath = getFileExportPath( - collectionExportPath, - videoExportName, - ); + const videoExportPath = `${collectionExportPath}/${videoExportName}`; log.info( `moving video file ${videoExportPath} to trash folder`, ); @@ -807,10 +801,7 @@ class ExportService { ); } } else { - const fileExportPath = getFileExportPath( - collectionExportPath, - fileExportName, - ); + const fileExportPath = `${collectionExportPath}/${fileExportName}`; const trashedFilePath = await getTrashedFileExportPath( exportDir, @@ -1071,7 +1062,7 @@ class ExportService { file, ); await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, fileExportName), + `${collectionExportPath}/${fileExportName}`, updatedFileStream, ); } catch (e) { From 1ffa905f9984a23ad0067f07cc6d48dc7939712a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 17:00:50 +0530 Subject: [PATCH 06/10] Inline 2 --- web/apps/photos/src/services/export/index.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index f5e06bc93d..9f61e05db3 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1110,7 +1110,7 @@ class ExportService { file, ); await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, imageExportName), + `${collectionExportPath}/${imageExportName}`, imageStream, ); @@ -1122,12 +1122,12 @@ class ExportService { ); try { await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, videoExportName), + `${collectionExportPath}/${videoExportName}`, videoStream, ); } catch (e) { await ensureElectron().deleteFile( - getFileExportPath(collectionExportPath, imageExportName), + `${collectionExportPath}/${imageExportName}`, ); throw e; } @@ -1192,8 +1192,3 @@ class ExportService { } export default new ExportService(); - -export const getFileExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${fileExportName}`; From 37cb2aaaf95d599e3d58076f36b0b7e123ff579b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 18:09:32 +0530 Subject: [PATCH 07/10] Refactor --- web/apps/photos/src/pages/_app.tsx | 38 +++----------------- web/apps/photos/src/services/export/index.ts | 26 +++++++++++++- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 06961d6c9b..c31256f138 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -52,7 +52,7 @@ import "photoswipe/dist/photoswipe.css"; import { createContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; -import exportService from "services/export"; +import exportService, { resumeExportsIfNeeded } from "services/export"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -64,7 +64,6 @@ import { NotificationAttributes, SetNotificationAttributes, } from "types/Notification"; -import { isExportInProgress } from "utils/export"; import { getMLSearchConfig, updateMLSearchConfig, @@ -214,37 +213,10 @@ export default function App({ Component, pageProps }: AppProps) { return; } const initExport = async () => { - try { - log.info("init export"); - const token = getToken(); - if (!token) { - log.info( - "User not logged in, not starting export continuous sync job", - ); - return; - } - await DownloadManager.init(APPS.PHOTOS, { token }); - const exportSettings = exportService.getExportSettings(); - if ( - !(await exportService.exportFolderExists( - exportSettings?.folder, - )) - ) { - return; - } - const exportRecord = await exportService.getExportRecord( - exportSettings.folder, - ); - if (exportSettings.continuousExport) { - exportService.enableContinuousExport(); - } - if (isExportInProgress(exportRecord.stage)) { - log.info("export was in progress, resuming"); - exportService.scheduleExport(); - } - } catch (e) { - log.error("init export failed", e); - } + const token = getToken(); + if (!token) return; + await DownloadManager.init(APPS.PHOTOS, { token }); + await resumeExportsIfNeeded(); }; initExport(); try { diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 9f61e05db3..05769e6c5e 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -42,6 +42,7 @@ import { getUnExportedFiles, getUniqueCollectionExportName, getUniqueFileExportName, + isExportInProgress, isLivePhotoExportName, parseLivePhotoExportName, } from "utils/export"; @@ -1191,4 +1192,27 @@ class ExportService { }; } -export default new ExportService(); +const exportService = new ExportService(); + +export default exportService; + +/** + * If there are any in-progress exports, or if continuous exports are enabled, + * resume them. + */ +export const resumeExportsIfNeeded = async () => { + const exportSettings = exportService.getExportSettings(); + if (!(await exportService.exportFolderExists(exportSettings?.folder))) { + return; + } + const exportRecord = await exportService.getExportRecord( + exportSettings.folder, + ); + if (exportSettings.continuousExport) { + exportService.enableContinuousExport(); + } + if (isExportInProgress(exportRecord.stage)) { + log.debug(() => "Resuming in-progress export"); + exportService.scheduleExport(); + } +}; From eb995f43544dde21263629d26c06eb1efbb7c07a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 18:20:40 +0530 Subject: [PATCH 08/10] Split --- web/apps/photos/src/services/export/index.ts | 267 ++++++++++++++-- .../photos/src/services/export/migration.ts | 14 +- web/apps/photos/src/utils/collection/index.ts | 2 +- web/apps/photos/src/utils/export/index.ts | 294 ------------------ web/apps/photos/src/utils/file/index.ts | 2 +- web/apps/photos/src/utils/native-fs.ts | 44 +++ 6 files changed, 297 insertions(+), 326 deletions(-) delete mode 100644 web/apps/photos/src/utils/export/index.ts create mode 100644 web/apps/photos/src/utils/native-fs.ts diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 05769e6c5e..f6a98e069e 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -3,55 +3,43 @@ import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage"; +import { formatDateTimeShort } from "@ente/shared/time/format"; import { User } from "@ente/shared/user/types"; import { sleep } from "@ente/shared/utils"; import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { ExportStage } from "constants/export"; +import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { + CollectionExportNames, ExportProgress, ExportRecord, ExportSettings, ExportUIUpdaters, + FileExportNames, } from "types/export"; import { EnteFile } from "types/file"; +import { Metadata } from "types/upload"; import { constructCollectionNameMap, getCollectionUserFacingName, getNonEmptyPersonalCollections, } from "utils/collection"; -import { - convertCollectionIDExportNameObjectToMap, - convertFileIDExportNameObjectToMap, - getCollectionExportedFiles, - getCollectionIDFromFileUID, - getDeletedExportedCollections, - getDeletedExportedFiles, - getExportRecordFileUID, - getFileMetadataExportPath, - getGoogleLikeMetadataFile, - getLivePhotoExportName, - getMetadataFileExportPath, - getMetadataFolderExportPath, - getRenamedExportedCollections, - getTrashedFileExportPath, - getUnExportedFiles, - getUniqueCollectionExportName, - getUniqueFileExportName, - isExportInProgress, - isLivePhotoExportName, - parseLivePhotoExportName, -} from "utils/export"; import { generateStreamFromArrayBuffer, getPersonalFiles, getUpdatedEXIFFileForDownload, mergeMetadata, + splitFilenameAndExtension, } from "utils/file"; +import { + ENTE_TRASH_FOLDER, + getUniqueCollectionExportName, + getUniqueFileExportName, +} from "utils/native-fs"; import { getAllLocalCollections } from "../collectionService"; import downloadManager from "../download"; import { getAllLocalFiles } from "../fileService"; @@ -1216,3 +1204,236 @@ export const resumeExportsIfNeeded = async () => { exportService.scheduleExport(); } }; + +export const getExportRecordFileUID = (file: EnteFile) => + `${file.id}_${file.collectionID}_${file.updationTime}`; + +export const getCollectionIDFromFileUID = (fileUID: string) => + Number(fileUID.split("_")[1]); + +const convertCollectionIDExportNameObjectToMap = ( + collectionExportNames: CollectionExportNames, +): Map => { + return new Map( + Object.entries(collectionExportNames ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }), + ); +}; + +const convertFileIDExportNameObjectToMap = ( + fileExportNames: FileExportNames, +): Map => { + return new Map( + Object.entries(fileExportNames ?? {}).map((e) => { + return [String(e[0]), String(e[1])]; + }), + ); +}; + +const getRenamedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( + exportRecord.collectionExportNames, + ); + const renamedCollections = collections.filter((collection) => { + if (collectionIDExportNameMap.has(collection.id)) { + const currentExportName = collectionIDExportNameMap.get( + collection.id, + ); + + const collectionExportName = + getCollectionUserFacingName(collection); + + if (currentExportName === collectionExportName) { + return false; + } + const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); + const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix + ? currentExportName.replace(/\(\d+\)$/, "") + : currentExportName; + + return ( + collectionExportName !== currentExportNameWithoutNumberedSuffix + ); + } + return false; + }); + return renamedCollections; +}; + +const getDeletedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const presentCollections = new Set( + collections.map((collection) => collection.id), + ); + const deletedExportedCollections = Object.keys( + exportRecord?.collectionExportNames, + ) + .map(Number) + .filter((collectionID) => { + if (!presentCollections.has(collectionID)) { + return true; + } + return false; + }); + return deletedExportedCollections; +}; + +const getUnExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.fileExportNames) { + return allFiles; + } + const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); + const unExportedFiles = allFiles.filter((file) => { + if (!exportedFiles.has(getExportRecordFileUID(file))) { + return true; + } + return false; + }); + return unExportedFiles; +}; + +const getDeletedExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const presentFileUIDs = new Set( + allFiles?.map((file) => getExportRecordFileUID(file)), + ); + const deletedExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + if (!presentFileUIDs.has(fileUID)) { + return true; + } + return false; + }); + return deletedExportedFiles; +}; + +const getCollectionExportedFiles = ( + exportRecord: ExportRecord, + collectionID: number, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const collectionExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + const fileCollectionID = Number(fileUID.split("_")[1]); + if (fileCollectionID === collectionID) { + return true; + } else { + return false; + } + }); + return collectionExportedFiles; +}; + +const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { + const metadata: Metadata = file.metadata; + const creationTime = Math.floor(metadata.creationTime / 1000000); + const modificationTime = Math.floor( + (metadata.modificationTime ?? metadata.creationTime) / 1000000, + ); + const captionValue: string = file?.pubMagicMetadata?.data?.caption; + return JSON.stringify( + { + title: fileExportName, + caption: captionValue, + creationTime: { + timestamp: creationTime, + formatted: formatDateTimeShort(creationTime * 1000), + }, + modificationTime: { + timestamp: modificationTime, + formatted: formatDateTimeShort(modificationTime * 1000), + }, + geoData: { + latitude: metadata.latitude, + longitude: metadata.longitude, + }, + }, + null, + 2, + ); +}; + +export const getMetadataFolderExportPath = (collectionExportPath: string) => + `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; + +const getFileMetadataExportPath = ( + collectionExportPath: string, + fileExportName: string, +) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; + +const getTrashedFileExportPath = async (exportDir: string, path: string) => { + const fileRelativePath = path.replace(`${exportDir}/`, ""); + let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; + let count = 1; + while (await exportService.exists(trashedFilePath)) { + const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); + if (trashedFilePathParts[1]) { + trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; + } else { + trashedFilePath = `${trashedFilePathParts[0]}(${count})`; + } + count++; + } + return trashedFilePath; +}; + +// if filepath is /home/user/Ente/Export/Collection1/1.jpg +// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json +const getMetadataFileExportPath = (filePath: string) => { + // extract filename and collection folder path + const filename = filePath.split("/").pop(); + const collectionExportPath = filePath.replace(`/${filename}`, ""); + return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; +}; + +export const getLivePhotoExportName = ( + imageExportName: string, + videoExportName: string, +) => + JSON.stringify({ + image: imageExportName, + video: videoExportName, + }); + +export const isLivePhotoExportName = (exportName: string) => { + try { + JSON.parse(exportName); + return true; + } catch (e) { + return false; + } +}; + +const parseLivePhotoExportName = ( + livePhotoExportName: string, +): { image: string; video: string } => { + const { image, video } = JSON.parse(livePhotoExportName); + return { image, video }; +}; + +const isExportInProgress = (exportStage: ExportStage) => + exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 812eafad4c..68ea9a4626 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -21,19 +21,19 @@ import { } from "types/export"; import { EnteFile } from "types/file"; import { getNonEmptyPersonalCollections } from "utils/collection"; -import { - getCollectionIDFromFileUID, - getExportRecordFileUID, - getLivePhotoExportName, - getMetadataFolderExportPath, - sanitizeName, -} from "utils/export"; import { splitFilenameAndExtension } from "utils/ffmpeg"; import { getIDBasedSortedFiles, getPersonalFiles, mergeMetadata, } from "utils/file"; +import { sanitizeName } from "utils/native-fs"; +import { + getCollectionIDFromFileUID, + getExportRecordFileUID, + getLivePhotoExportName, + getMetadataFolderExportPath, +} from "."; import exportService from "./index"; export async function migrateExport( diff --git a/web/apps/photos/src/utils/collection/index.ts b/web/apps/photos/src/utils/collection/index.ts index 3b0f955e09..581523828d 100644 --- a/web/apps/photos/src/utils/collection/index.ts +++ b/web/apps/photos/src/utils/collection/index.ts @@ -42,9 +42,9 @@ import { import { EnteFile } from "types/file"; import { SetFilesDownloadProgressAttributes } from "types/gallery"; import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata"; -import { getUniqueCollectionExportName } from "utils/export"; import { downloadFilesWithProgress } from "utils/file"; import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueCollectionExportName } from "utils/native-fs"; export enum COLLECTION_OPS_TYPE { ADD, diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts deleted file mode 100644 index 76f1e1bb03..0000000000 --- a/web/apps/photos/src/utils/export/index.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { formatDateTimeShort } from "@ente/shared/time/format"; -import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; -import sanitize from "sanitize-filename"; -import exportService from "services/export"; -import { Collection } from "types/collection"; -import { - CollectionExportNames, - ExportRecord, - FileExportNames, -} from "types/export"; -import { EnteFile } from "types/file"; -import { Metadata } from "types/upload"; -import { getCollectionUserFacingName } from "utils/collection"; -import { splitFilenameAndExtension } from "utils/file"; - -export const ENTE_TRASH_FOLDER = "Trash"; - -export const getExportRecordFileUID = (file: EnteFile) => - `${file.id}_${file.collectionID}_${file.updationTime}`; - -export const getCollectionIDFromFileUID = (fileUID: string) => - Number(fileUID.split("_")[1]); - -export const convertCollectionIDExportNameObjectToMap = ( - collectionExportNames: CollectionExportNames, -): Map => { - return new Map( - Object.entries(collectionExportNames ?? {}).map((e) => { - return [Number(e[0]), String(e[1])]; - }), - ); -}; - -export const convertFileIDExportNameObjectToMap = ( - fileExportNames: FileExportNames, -): Map => { - return new Map( - Object.entries(fileExportNames ?? {}).map((e) => { - return [String(e[0]), String(e[1])]; - }), - ); -}; - -export const getRenamedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( - exportRecord.collectionExportNames, - ); - const renamedCollections = collections.filter((collection) => { - if (collectionIDExportNameMap.has(collection.id)) { - const currentExportName = collectionIDExportNameMap.get( - collection.id, - ); - - const collectionExportName = - getCollectionUserFacingName(collection); - - if (currentExportName === collectionExportName) { - return false; - } - const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); - const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix - ? currentExportName.replace(/\(\d+\)$/, "") - : currentExportName; - - return ( - collectionExportName !== currentExportNameWithoutNumberedSuffix - ); - } - return false; - }); - return renamedCollections; -}; - -export const getDeletedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const presentCollections = new Set( - collections.map((collection) => collection.id), - ); - const deletedExportedCollections = Object.keys( - exportRecord?.collectionExportNames, - ) - .map(Number) - .filter((collectionID) => { - if (!presentCollections.has(collectionID)) { - return true; - } - return false; - }); - return deletedExportedCollections; -}; - -export const getUnExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.fileExportNames) { - return allFiles; - } - const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); - const unExportedFiles = allFiles.filter((file) => { - if (!exportedFiles.has(getExportRecordFileUID(file))) { - return true; - } - return false; - }); - return unExportedFiles; -}; - -export const getDeletedExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const presentFileUIDs = new Set( - allFiles?.map((file) => getExportRecordFileUID(file)), - ); - const deletedExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - if (!presentFileUIDs.has(fileUID)) { - return true; - } - return false; - }); - return deletedExportedFiles; -}; - -export const getCollectionExportedFiles = ( - exportRecord: ExportRecord, - collectionID: number, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const collectionExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - const fileCollectionID = Number(fileUID.split("_")[1]); - if (fileCollectionID === collectionID) { - return true; - } else { - return false; - } - }); - return collectionExportedFiles; -}; - -export const getGoogleLikeMetadataFile = ( - fileExportName: string, - file: EnteFile, -) => { - const metadata: Metadata = file.metadata; - const creationTime = Math.floor(metadata.creationTime / 1000000); - const modificationTime = Math.floor( - (metadata.modificationTime ?? metadata.creationTime) / 1000000, - ); - const captionValue: string = file?.pubMagicMetadata?.data?.caption; - return JSON.stringify( - { - title: fileExportName, - caption: captionValue, - creationTime: { - timestamp: creationTime, - formatted: formatDateTimeShort(creationTime * 1000), - }, - modificationTime: { - timestamp: modificationTime, - formatted: formatDateTimeShort(modificationTime * 1000), - }, - geoData: { - latitude: metadata.latitude, - longitude: metadata.longitude, - }, - }, - null, - 2, - ); -}; - -export const sanitizeName = (name: string) => - sanitize(name, { replacement: "_" }); - -export const getUniqueCollectionExportName = async ( - dir: string, - collectionName: string, -): Promise => { - let collectionExportName = sanitizeName(collectionName); - let count = 1; - while ( - (await exportService.exists(`${dir}/${collectionExportName}`)) || - collectionExportName === ENTE_TRASH_FOLDER - ) { - collectionExportName = `${sanitizeName(collectionName)}(${count})`; - count++; - } - return collectionExportName; -}; - -export const getMetadataFolderExportPath = (collectionExportPath: string) => - `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; - -export const getUniqueFileExportName = async ( - collectionExportPath: string, - filename: string, -) => { - let fileExportName = sanitizeName(filename); - let count = 1; - while ( - await exportService.exists(`${collectionExportPath}/${fileExportName}`) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileExportName = `${filenameParts[0]}(${count})`; - } - count++; - } - return fileExportName; -}; - -export const getFileMetadataExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; - -export const getTrashedFileExportPath = async ( - exportDir: string, - path: string, -) => { - const fileRelativePath = path.replace(`${exportDir}/`, ""); - let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; - let count = 1; - while (await exportService.exists(trashedFilePath)) { - const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); - if (trashedFilePathParts[1]) { - trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; - } else { - trashedFilePath = `${trashedFilePathParts[0]}(${count})`; - } - count++; - } - return trashedFilePath; -}; - -// if filepath is /home/user/Ente/Export/Collection1/1.jpg -// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json -export const getMetadataFileExportPath = (filePath: string) => { - // extract filename and collection folder path - const filename = filePath.split("/").pop(); - const collectionExportPath = filePath.replace(`/${filename}`, ""); - return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; -}; - -export const getLivePhotoExportName = ( - imageExportName: string, - videoExportName: string, -) => - JSON.stringify({ - image: imageExportName, - video: videoExportName, - }); - -export const isLivePhotoExportName = (exportName: string) => { - try { - JSON.parse(exportName); - return true; - } catch (e) { - return false; - } -}; - -export const parseLivePhotoExportName = ( - livePhotoExportName: string, -): { image: string; video: string } => { - const { image, video } = JSON.parse(livePhotoExportName); - return { image, video }; -}; - -export const isExportInProgress = (exportStage: ExportStage) => - exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index d615ef707e..cd432ecbe0 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -51,8 +51,8 @@ import { } from "types/gallery"; import { VISIBILITY_STATE } from "types/magicMetadata"; import { FileTypeInfo } from "types/upload"; -import { getUniqueFileExportName } from "utils/export"; import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueFileExportName } from "utils/native-fs"; const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000; diff --git a/web/apps/photos/src/utils/native-fs.ts b/web/apps/photos/src/utils/native-fs.ts new file mode 100644 index 0000000000..4173aa7ac0 --- /dev/null +++ b/web/apps/photos/src/utils/native-fs.ts @@ -0,0 +1,44 @@ +import sanitize from "sanitize-filename"; +import exportService from "services/export"; +import { splitFilenameAndExtension } from "utils/file"; + +export const ENTE_TRASH_FOLDER = "Trash"; + +export const sanitizeName = (name: string) => + sanitize(name, { replacement: "_" }); + +export const getUniqueCollectionExportName = async ( + dir: string, + collectionName: string, +): Promise => { + let collectionExportName = sanitizeName(collectionName); + let count = 1; + while ( + (await exportService.exists(`${dir}/${collectionExportName}`)) || + collectionExportName === ENTE_TRASH_FOLDER + ) { + collectionExportName = `${sanitizeName(collectionName)}(${count})`; + count++; + } + return collectionExportName; +}; + +export const getUniqueFileExportName = async ( + collectionExportPath: string, + filename: string, +) => { + let fileExportName = sanitizeName(filename); + let count = 1; + while ( + await exportService.exists(`${collectionExportPath}/${fileExportName}`) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileExportName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileExportName; +}; From 63a2ca76066f92ffdf57db47b2a98eb7e3b1191d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 18:24:57 +0530 Subject: [PATCH 09/10] Fix lint --- web/apps/photos/src/constants/export.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/apps/photos/src/constants/export.ts b/web/apps/photos/src/constants/export.ts index 87be9f2384..692ee3843f 100644 --- a/web/apps/photos/src/constants/export.ts +++ b/web/apps/photos/src/constants/export.ts @@ -1,6 +1,5 @@ export const ENTE_METADATA_FOLDER = "metadata"; - export enum ExportStage { INIT = 0, MIGRATION = 1, From 4f764dc77c9393a0e0491b419b69d179f9cdec5b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 18:27:40 +0530 Subject: [PATCH 10/10] Consolidate constants --- .../photos/src/components/ExportInProgress.tsx | 2 +- web/apps/photos/src/components/ExportModal.tsx | 3 +-- web/apps/photos/src/constants/export.ts | 12 ------------ web/apps/photos/src/services/export/index.ts | 14 +++++++++++++- web/apps/photos/src/services/export/migration.ts | 2 +- web/apps/photos/src/types/export/index.ts | 2 +- web/apps/photos/src/utils/upload/index.ts | 2 +- 7 files changed, 18 insertions(+), 19 deletions(-) delete mode 100644 web/apps/photos/src/constants/export.ts diff --git a/web/apps/photos/src/components/ExportInProgress.tsx b/web/apps/photos/src/components/ExportInProgress.tsx index ce2da895c0..280ae52d4d 100644 --- a/web/apps/photos/src/components/ExportInProgress.tsx +++ b/web/apps/photos/src/components/ExportInProgress.tsx @@ -10,9 +10,9 @@ import { LinearProgress, styled, } from "@mui/material"; -import { ExportStage } from "constants/export"; import { t } from "i18next"; import { Trans } from "react-i18next"; +import { ExportStage } from "services/export"; import { ExportProgress } from "types/export"; export const ComfySpan = styled("span")` diff --git a/web/apps/photos/src/components/ExportModal.tsx b/web/apps/photos/src/components/ExportModal.tsx index 877dee90f1..159c872e44 100644 --- a/web/apps/photos/src/components/ExportModal.tsx +++ b/web/apps/photos/src/components/ExportModal.tsx @@ -14,12 +14,11 @@ import { Switch, Typography, } from "@mui/material"; -import { ExportStage } from "constants/export"; import { t } from "i18next"; import isElectron from "is-electron"; import { AppContext } from "pages/_app"; import { useContext, useEffect, useState } from "react"; -import exportService from "services/export"; +import exportService, { ExportStage } from "services/export"; import { ExportProgress, ExportSettings } from "types/export"; import { EnteFile } from "types/file"; import { getExportDirectoryDoesNotExistMessage } from "utils/ui"; diff --git a/web/apps/photos/src/constants/export.ts b/web/apps/photos/src/constants/export.ts deleted file mode 100644 index 692ee3843f..0000000000 --- a/web/apps/photos/src/constants/export.ts +++ /dev/null @@ -1,12 +0,0 @@ -export const ENTE_METADATA_FOLDER = "metadata"; - -export enum ExportStage { - INIT = 0, - MIGRATION = 1, - STARTING = 2, - EXPORTING_FILES = 3, - TRASHING_DELETED_FILES = 4, - RENAMING_COLLECTION_FOLDERS = 5, - TRASHING_DELETED_COLLECTIONS = 6, - FINISHED = 7, -} diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index f6a98e069e..419db15876 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -10,7 +10,6 @@ import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { @@ -50,6 +49,19 @@ const EXPORT_RECORD_FILE_NAME = "export_status.json"; export const ENTE_EXPORT_DIRECTORY = "ente Photos"; +export const ENTE_METADATA_FOLDER = "metadata"; + +export enum ExportStage { + INIT = 0, + MIGRATION = 1, + STARTING = 2, + EXPORTING_FILES = 3, + TRASHING_DELETED_FILES = 4, + RENAMING_COLLECTION_FOLDERS = 5, + TRASHING_DELETED_COLLECTIONS = 6, + FINISHED = 7, +} + export const NULL_EXPORT_RECORD: ExportRecord = { version: 3, lastAttemptTimestamp: null, diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 68ea9a4626..49265cf348 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -2,7 +2,6 @@ import log from "@/next/log"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import { User } from "@ente/shared/user/types"; import { sleep } from "@ente/shared/utils"; -import { ENTE_METADATA_FOLDER } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { getLocalCollections } from "services/collectionService"; import downloadManager from "services/download"; @@ -29,6 +28,7 @@ import { } from "utils/file"; import { sanitizeName } from "utils/native-fs"; import { + ENTE_METADATA_FOLDER, getCollectionIDFromFileUID, getExportRecordFileUID, getLivePhotoExportName, diff --git a/web/apps/photos/src/types/export/index.ts b/web/apps/photos/src/types/export/index.ts index ce85f32fd1..64ef249eda 100644 --- a/web/apps/photos/src/types/export/index.ts +++ b/web/apps/photos/src/types/export/index.ts @@ -1,4 +1,4 @@ -import { ExportStage } from "constants/export"; +import type { ExportStage } from "services/export"; import { EnteFile } from "types/file"; export interface ExportProgress { diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 6cce03aa9d..708ec5dcf7 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,4 +1,3 @@ -import { ENTE_METADATA_FOLDER } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { A_SEC_IN_MICROSECONDS, @@ -6,6 +5,7 @@ import { PICKED_UPLOAD_TYPE, } from "constants/upload"; import isElectron from "is-electron"; +import { ENTE_METADATA_FOLDER } from "services/export"; import { EnteFile } from "types/file"; import { ElectronFile,