From eb995f43544dde21263629d26c06eb1efbb7c07a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 13 Apr 2024 18:20:40 +0530 Subject: [PATCH] Split --- web/apps/photos/src/services/export/index.ts | 267 ++++++++++++++-- .../photos/src/services/export/migration.ts | 14 +- web/apps/photos/src/utils/collection/index.ts | 2 +- web/apps/photos/src/utils/export/index.ts | 294 ------------------ web/apps/photos/src/utils/file/index.ts | 2 +- web/apps/photos/src/utils/native-fs.ts | 44 +++ 6 files changed, 297 insertions(+), 326 deletions(-) delete mode 100644 web/apps/photos/src/utils/export/index.ts create mode 100644 web/apps/photos/src/utils/native-fs.ts diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 05769e6c5e..f6a98e069e 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -3,55 +3,43 @@ import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage"; +import { formatDateTimeShort } from "@ente/shared/time/format"; import { User } from "@ente/shared/user/types"; import { sleep } from "@ente/shared/utils"; import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { ExportStage } from "constants/export"; +import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { + CollectionExportNames, ExportProgress, ExportRecord, ExportSettings, ExportUIUpdaters, + FileExportNames, } from "types/export"; import { EnteFile } from "types/file"; +import { Metadata } from "types/upload"; import { constructCollectionNameMap, getCollectionUserFacingName, getNonEmptyPersonalCollections, } from "utils/collection"; -import { - convertCollectionIDExportNameObjectToMap, - convertFileIDExportNameObjectToMap, - getCollectionExportedFiles, - getCollectionIDFromFileUID, - getDeletedExportedCollections, - getDeletedExportedFiles, - getExportRecordFileUID, - getFileMetadataExportPath, - getGoogleLikeMetadataFile, - getLivePhotoExportName, - getMetadataFileExportPath, - getMetadataFolderExportPath, - getRenamedExportedCollections, - getTrashedFileExportPath, - getUnExportedFiles, - getUniqueCollectionExportName, - getUniqueFileExportName, - isExportInProgress, - isLivePhotoExportName, - parseLivePhotoExportName, -} from "utils/export"; import { generateStreamFromArrayBuffer, getPersonalFiles, getUpdatedEXIFFileForDownload, mergeMetadata, + splitFilenameAndExtension, } from "utils/file"; +import { + ENTE_TRASH_FOLDER, + getUniqueCollectionExportName, + getUniqueFileExportName, +} from "utils/native-fs"; import { getAllLocalCollections } from "../collectionService"; import downloadManager from "../download"; import { getAllLocalFiles } from "../fileService"; @@ -1216,3 +1204,236 @@ export const resumeExportsIfNeeded = async () => { exportService.scheduleExport(); } }; + +export const getExportRecordFileUID = (file: EnteFile) => + `${file.id}_${file.collectionID}_${file.updationTime}`; + +export const getCollectionIDFromFileUID = (fileUID: string) => + Number(fileUID.split("_")[1]); + +const convertCollectionIDExportNameObjectToMap = ( + collectionExportNames: CollectionExportNames, +): Map => { + return new Map( + Object.entries(collectionExportNames ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }), + ); +}; + +const convertFileIDExportNameObjectToMap = ( + fileExportNames: FileExportNames, +): Map => { + return new Map( + Object.entries(fileExportNames ?? {}).map((e) => { + return [String(e[0]), String(e[1])]; + }), + ); +}; + +const getRenamedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( + exportRecord.collectionExportNames, + ); + const renamedCollections = collections.filter((collection) => { + if (collectionIDExportNameMap.has(collection.id)) { + const currentExportName = collectionIDExportNameMap.get( + collection.id, + ); + + const collectionExportName = + getCollectionUserFacingName(collection); + + if (currentExportName === collectionExportName) { + return false; + } + const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); + const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix + ? currentExportName.replace(/\(\d+\)$/, "") + : currentExportName; + + return ( + collectionExportName !== currentExportNameWithoutNumberedSuffix + ); + } + return false; + }); + return renamedCollections; +}; + +const getDeletedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const presentCollections = new Set( + collections.map((collection) => collection.id), + ); + const deletedExportedCollections = Object.keys( + exportRecord?.collectionExportNames, + ) + .map(Number) + .filter((collectionID) => { + if (!presentCollections.has(collectionID)) { + return true; + } + return false; + }); + return deletedExportedCollections; +}; + +const getUnExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.fileExportNames) { + return allFiles; + } + const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); + const unExportedFiles = allFiles.filter((file) => { + if (!exportedFiles.has(getExportRecordFileUID(file))) { + return true; + } + return false; + }); + return unExportedFiles; +}; + +const getDeletedExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const presentFileUIDs = new Set( + allFiles?.map((file) => getExportRecordFileUID(file)), + ); + const deletedExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + if (!presentFileUIDs.has(fileUID)) { + return true; + } + return false; + }); + return deletedExportedFiles; +}; + +const getCollectionExportedFiles = ( + exportRecord: ExportRecord, + collectionID: number, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const collectionExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + const fileCollectionID = Number(fileUID.split("_")[1]); + if (fileCollectionID === collectionID) { + return true; + } else { + return false; + } + }); + return collectionExportedFiles; +}; + +const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { + const metadata: Metadata = file.metadata; + const creationTime = Math.floor(metadata.creationTime / 1000000); + const modificationTime = Math.floor( + (metadata.modificationTime ?? metadata.creationTime) / 1000000, + ); + const captionValue: string = file?.pubMagicMetadata?.data?.caption; + return JSON.stringify( + { + title: fileExportName, + caption: captionValue, + creationTime: { + timestamp: creationTime, + formatted: formatDateTimeShort(creationTime * 1000), + }, + modificationTime: { + timestamp: modificationTime, + formatted: formatDateTimeShort(modificationTime * 1000), + }, + geoData: { + latitude: metadata.latitude, + longitude: metadata.longitude, + }, + }, + null, + 2, + ); +}; + +export const getMetadataFolderExportPath = (collectionExportPath: string) => + `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; + +const getFileMetadataExportPath = ( + collectionExportPath: string, + fileExportName: string, +) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; + +const getTrashedFileExportPath = async (exportDir: string, path: string) => { + const fileRelativePath = path.replace(`${exportDir}/`, ""); + let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; + let count = 1; + while (await exportService.exists(trashedFilePath)) { + const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); + if (trashedFilePathParts[1]) { + trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; + } else { + trashedFilePath = `${trashedFilePathParts[0]}(${count})`; + } + count++; + } + return trashedFilePath; +}; + +// if filepath is /home/user/Ente/Export/Collection1/1.jpg +// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json +const getMetadataFileExportPath = (filePath: string) => { + // extract filename and collection folder path + const filename = filePath.split("/").pop(); + const collectionExportPath = filePath.replace(`/${filename}`, ""); + return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; +}; + +export const getLivePhotoExportName = ( + imageExportName: string, + videoExportName: string, +) => + JSON.stringify({ + image: imageExportName, + video: videoExportName, + }); + +export const isLivePhotoExportName = (exportName: string) => { + try { + JSON.parse(exportName); + return true; + } catch (e) { + return false; + } +}; + +const parseLivePhotoExportName = ( + livePhotoExportName: string, +): { image: string; video: string } => { + const { image, video } = JSON.parse(livePhotoExportName); + return { image, video }; +}; + +const isExportInProgress = (exportStage: ExportStage) => + exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 812eafad4c..68ea9a4626 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -21,19 +21,19 @@ import { } from "types/export"; import { EnteFile } from "types/file"; import { getNonEmptyPersonalCollections } from "utils/collection"; -import { - getCollectionIDFromFileUID, - getExportRecordFileUID, - getLivePhotoExportName, - getMetadataFolderExportPath, - sanitizeName, -} from "utils/export"; import { splitFilenameAndExtension } from "utils/ffmpeg"; import { getIDBasedSortedFiles, getPersonalFiles, mergeMetadata, } from "utils/file"; +import { sanitizeName } from "utils/native-fs"; +import { + getCollectionIDFromFileUID, + getExportRecordFileUID, + getLivePhotoExportName, + getMetadataFolderExportPath, +} from "."; import exportService from "./index"; export async function migrateExport( diff --git a/web/apps/photos/src/utils/collection/index.ts b/web/apps/photos/src/utils/collection/index.ts index 3b0f955e09..581523828d 100644 --- a/web/apps/photos/src/utils/collection/index.ts +++ b/web/apps/photos/src/utils/collection/index.ts @@ -42,9 +42,9 @@ import { import { EnteFile } from "types/file"; import { SetFilesDownloadProgressAttributes } from "types/gallery"; import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata"; -import { getUniqueCollectionExportName } from "utils/export"; import { downloadFilesWithProgress } from "utils/file"; import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueCollectionExportName } from "utils/native-fs"; export enum COLLECTION_OPS_TYPE { ADD, diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts deleted file mode 100644 index 76f1e1bb03..0000000000 --- a/web/apps/photos/src/utils/export/index.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { formatDateTimeShort } from "@ente/shared/time/format"; -import { ENTE_METADATA_FOLDER, ExportStage } from "constants/export"; -import sanitize from "sanitize-filename"; -import exportService from "services/export"; -import { Collection } from "types/collection"; -import { - CollectionExportNames, - ExportRecord, - FileExportNames, -} from "types/export"; -import { EnteFile } from "types/file"; -import { Metadata } from "types/upload"; -import { getCollectionUserFacingName } from "utils/collection"; -import { splitFilenameAndExtension } from "utils/file"; - -export const ENTE_TRASH_FOLDER = "Trash"; - -export const getExportRecordFileUID = (file: EnteFile) => - `${file.id}_${file.collectionID}_${file.updationTime}`; - -export const getCollectionIDFromFileUID = (fileUID: string) => - Number(fileUID.split("_")[1]); - -export const convertCollectionIDExportNameObjectToMap = ( - collectionExportNames: CollectionExportNames, -): Map => { - return new Map( - Object.entries(collectionExportNames ?? {}).map((e) => { - return [Number(e[0]), String(e[1])]; - }), - ); -}; - -export const convertFileIDExportNameObjectToMap = ( - fileExportNames: FileExportNames, -): Map => { - return new Map( - Object.entries(fileExportNames ?? {}).map((e) => { - return [String(e[0]), String(e[1])]; - }), - ); -}; - -export const getRenamedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( - exportRecord.collectionExportNames, - ); - const renamedCollections = collections.filter((collection) => { - if (collectionIDExportNameMap.has(collection.id)) { - const currentExportName = collectionIDExportNameMap.get( - collection.id, - ); - - const collectionExportName = - getCollectionUserFacingName(collection); - - if (currentExportName === collectionExportName) { - return false; - } - const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); - const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix - ? currentExportName.replace(/\(\d+\)$/, "") - : currentExportName; - - return ( - collectionExportName !== currentExportNameWithoutNumberedSuffix - ); - } - return false; - }); - return renamedCollections; -}; - -export const getDeletedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const presentCollections = new Set( - collections.map((collection) => collection.id), - ); - const deletedExportedCollections = Object.keys( - exportRecord?.collectionExportNames, - ) - .map(Number) - .filter((collectionID) => { - if (!presentCollections.has(collectionID)) { - return true; - } - return false; - }); - return deletedExportedCollections; -}; - -export const getUnExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.fileExportNames) { - return allFiles; - } - const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); - const unExportedFiles = allFiles.filter((file) => { - if (!exportedFiles.has(getExportRecordFileUID(file))) { - return true; - } - return false; - }); - return unExportedFiles; -}; - -export const getDeletedExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const presentFileUIDs = new Set( - allFiles?.map((file) => getExportRecordFileUID(file)), - ); - const deletedExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - if (!presentFileUIDs.has(fileUID)) { - return true; - } - return false; - }); - return deletedExportedFiles; -}; - -export const getCollectionExportedFiles = ( - exportRecord: ExportRecord, - collectionID: number, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const collectionExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - const fileCollectionID = Number(fileUID.split("_")[1]); - if (fileCollectionID === collectionID) { - return true; - } else { - return false; - } - }); - return collectionExportedFiles; -}; - -export const getGoogleLikeMetadataFile = ( - fileExportName: string, - file: EnteFile, -) => { - const metadata: Metadata = file.metadata; - const creationTime = Math.floor(metadata.creationTime / 1000000); - const modificationTime = Math.floor( - (metadata.modificationTime ?? metadata.creationTime) / 1000000, - ); - const captionValue: string = file?.pubMagicMetadata?.data?.caption; - return JSON.stringify( - { - title: fileExportName, - caption: captionValue, - creationTime: { - timestamp: creationTime, - formatted: formatDateTimeShort(creationTime * 1000), - }, - modificationTime: { - timestamp: modificationTime, - formatted: formatDateTimeShort(modificationTime * 1000), - }, - geoData: { - latitude: metadata.latitude, - longitude: metadata.longitude, - }, - }, - null, - 2, - ); -}; - -export const sanitizeName = (name: string) => - sanitize(name, { replacement: "_" }); - -export const getUniqueCollectionExportName = async ( - dir: string, - collectionName: string, -): Promise => { - let collectionExportName = sanitizeName(collectionName); - let count = 1; - while ( - (await exportService.exists(`${dir}/${collectionExportName}`)) || - collectionExportName === ENTE_TRASH_FOLDER - ) { - collectionExportName = `${sanitizeName(collectionName)}(${count})`; - count++; - } - return collectionExportName; -}; - -export const getMetadataFolderExportPath = (collectionExportPath: string) => - `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; - -export const getUniqueFileExportName = async ( - collectionExportPath: string, - filename: string, -) => { - let fileExportName = sanitizeName(filename); - let count = 1; - while ( - await exportService.exists(`${collectionExportPath}/${fileExportName}`) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileExportName = `${filenameParts[0]}(${count})`; - } - count++; - } - return fileExportName; -}; - -export const getFileMetadataExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; - -export const getTrashedFileExportPath = async ( - exportDir: string, - path: string, -) => { - const fileRelativePath = path.replace(`${exportDir}/`, ""); - let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; - let count = 1; - while (await exportService.exists(trashedFilePath)) { - const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); - if (trashedFilePathParts[1]) { - trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; - } else { - trashedFilePath = `${trashedFilePathParts[0]}(${count})`; - } - count++; - } - return trashedFilePath; -}; - -// if filepath is /home/user/Ente/Export/Collection1/1.jpg -// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json -export const getMetadataFileExportPath = (filePath: string) => { - // extract filename and collection folder path - const filename = filePath.split("/").pop(); - const collectionExportPath = filePath.replace(`/${filename}`, ""); - return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; -}; - -export const getLivePhotoExportName = ( - imageExportName: string, - videoExportName: string, -) => - JSON.stringify({ - image: imageExportName, - video: videoExportName, - }); - -export const isLivePhotoExportName = (exportName: string) => { - try { - JSON.parse(exportName); - return true; - } catch (e) { - return false; - } -}; - -export const parseLivePhotoExportName = ( - livePhotoExportName: string, -): { image: string; video: string } => { - const { image, video } = JSON.parse(livePhotoExportName); - return { image, video }; -}; - -export const isExportInProgress = (exportStage: ExportStage) => - exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index d615ef707e..cd432ecbe0 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -51,8 +51,8 @@ import { } from "types/gallery"; import { VISIBILITY_STATE } from "types/magicMetadata"; import { FileTypeInfo } from "types/upload"; -import { getUniqueFileExportName } from "utils/export"; import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueFileExportName } from "utils/native-fs"; const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000; diff --git a/web/apps/photos/src/utils/native-fs.ts b/web/apps/photos/src/utils/native-fs.ts new file mode 100644 index 0000000000..4173aa7ac0 --- /dev/null +++ b/web/apps/photos/src/utils/native-fs.ts @@ -0,0 +1,44 @@ +import sanitize from "sanitize-filename"; +import exportService from "services/export"; +import { splitFilenameAndExtension } from "utils/file"; + +export const ENTE_TRASH_FOLDER = "Trash"; + +export const sanitizeName = (name: string) => + sanitize(name, { replacement: "_" }); + +export const getUniqueCollectionExportName = async ( + dir: string, + collectionName: string, +): Promise => { + let collectionExportName = sanitizeName(collectionName); + let count = 1; + while ( + (await exportService.exists(`${dir}/${collectionExportName}`)) || + collectionExportName === ENTE_TRASH_FOLDER + ) { + collectionExportName = `${sanitizeName(collectionName)}(${count})`; + count++; + } + return collectionExportName; +}; + +export const getUniqueFileExportName = async ( + collectionExportPath: string, + filename: string, +) => { + let fileExportName = sanitizeName(filename); + let count = 1; + while ( + await exportService.exists(`${collectionExportPath}/${fileExportName}`) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileExportName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileExportName; +};