diff --git a/rust/src/commands/export.rs b/rust/src/commands/export.rs index ed3651a094..0d58b1411d 100644 --- a/rust/src/commands/export.rs +++ b/rust/src/commands/export.rs @@ -19,6 +19,86 @@ use std::path::{Path, PathBuf}; use tokio::fs; use tokio::io::AsyncWriteExt; +/// Information about a file already on disk, loaded from metadata +#[derive(Debug, Clone)] +struct ExistingFile { + /// Path to the actual file on disk + file_path: PathBuf, + /// Path to the metadata JSON file + meta_path: PathBuf, +} + +/// Load existing file metadata for a specific album +async fn load_album_metadata( + export_path: &Path, + album_name: &str, +) -> Result> { + let mut existing_files = HashMap::new(); + + let meta_dir = export_path.join(album_name).join(".meta"); + if !meta_dir.exists() { + return Ok(existing_files); + } + + let mut entries = match fs::read_dir(&meta_dir).await { + Ok(entries) => entries, + Err(e) => { + log::warn!("Failed to read metadata directory {:?}: {}", meta_dir, e); + return Ok(existing_files); + } + }; + + while let Some(entry) = entries.next_entry().await? { + let meta_path = entry.path(); + + // Skip non-JSON files and album_meta.json + if meta_path.extension().is_none_or(|ext| ext != "json") { + continue; + } + + if meta_path + .file_name() + .is_some_and(|name| name == "album_meta.json") + { + continue; + } + + // Read and parse metadata file + let json_content = match fs::read_to_string(&meta_path).await { + Ok(content) => content, + Err(e) => { + log::warn!("Failed to read metadata file {:?}: {}", meta_path, e); + continue; + } + }; + + let disk_metadata: DiskFileMetadata = match serde_json::from_str(&json_content) { + Ok(metadata) => metadata, + Err(e) => { + log::warn!("Failed to parse metadata file {:?}: {}", meta_path, e); + continue; + } + }; + + // Check if the actual file exists + for filename in &disk_metadata.info.file_names { + let file_path = export_path.join(album_name).join(filename); + if file_path.exists() { + existing_files.insert( + disk_metadata.info.id, + ExistingFile { + file_path, + meta_path: meta_path.clone(), + }, + ); + break; // Only need one existing file per ID + } + } + } + + Ok(existing_files) +} + pub async fn run_export(account_email: Option, filter: ExportFilter) -> Result<()> { // Initialize crypto crypto_init()?; @@ -164,6 +244,9 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil // Track file indices per album for unique metadata filenames let mut album_file_indices: HashMap = HashMap::new(); + // Track existing files per album (loaded on demand) + let mut album_existing_files: HashMap> = HashMap::new(); + // Get stored secrets let secrets = storage .accounts() @@ -437,19 +520,6 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil None }; - // Check if we've already exported a file with this hash - if let Some(hash) = content_hash - && let Some(existing_path) = exported_hashes.get(hash) - { - log::info!( - "Skipping duplicate file {} (same hash as {})", - file.id, - existing_path.display() - ); - skipped_files += 1; - continue; - } - // Generate export path with original filename from metadata let file_path = generate_export_path( export_path, @@ -459,9 +529,98 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil pub_magic_metadata.as_ref(), )?; - // Skip if file already exists on disk - if file_path.exists() { - log::debug!("File already exists: {file_path:?}"); + // Determine album folder + let album_folder = if let Some(ref name) = collection.name + && !name.is_empty() + { + sanitize_album_name(name) + } else { + "Uncategorized".to_string() + }; + + // Load existing files for this album if not already loaded + if !album_existing_files.contains_key(&album_folder) { + let existing = load_album_metadata(export_path, &album_folder).await?; + log::debug!( + "Loaded {} existing files for album {}", + existing.len(), + album_folder + ); + album_existing_files.insert(album_folder.clone(), existing); + } + + // Check if this file already exists in the album by ID (for rename detection) + let existing_files = album_existing_files.get_mut(&album_folder).unwrap(); + if let Some(existing) = existing_files.get(&file.id) { + if existing.file_path == file_path { + // File exists at the same path - no rename needed + log::debug!( + "File {} already exists at correct path: {:?}", + file.id, + file_path + ); + skipped_files += 1; + + // Add to hash map even for existing files to prevent duplicates + if let Some(hash) = content_hash { + exported_hashes.insert(hash.clone(), file_path.clone()); + } + continue; + } else { + // File exists at different path - it was renamed + log::info!( + "File {} renamed from {:?} to {:?}", + file.id, + existing.file_path, + file_path + ); + + // Remove old file + if existing.file_path.exists() { + log::debug!("Removing old file: {:?}", existing.file_path); + fs::remove_file(&existing.file_path).await.ok(); + } + + // For live photos, also remove the MOV component + let is_live_photo = metadata + .as_ref() + .map(|m| m.is_live_photo()) + .unwrap_or(false); + + if is_live_photo { + // Try to remove old MOV file + // The MOV file has the same base name but with .MOV extension + let old_mov_path = existing.file_path.with_extension("MOV"); + if old_mov_path.exists() { + log::debug!("Removing old live photo MOV component: {:?}", old_mov_path); + fs::remove_file(&old_mov_path).await.ok(); + } + + // Also try lowercase .mov + let old_mov_path_lower = existing.file_path.with_extension("mov"); + if old_mov_path_lower.exists() && old_mov_path_lower != old_mov_path { + log::debug!( + "Removing old live photo mov component: {:?}", + old_mov_path_lower + ); + fs::remove_file(&old_mov_path_lower).await.ok(); + } + } + + // Remove old metadata + if existing.meta_path.exists() { + log::debug!("Removing old metadata: {:?}", existing.meta_path); + fs::remove_file(&existing.meta_path).await.ok(); + } + + // Remove from tracking so we don't try to remove it again + existing_files.remove(&file.id); + + // Continue to re-export with new name + } + } else if file_path.exists() { + // File exists but not tracked by ID (old export or hash collision) + log::debug!("File already exists (not tracked by ID): {file_path:?}"); skipped_files += 1; // Add to hash map even for existing files to prevent duplicates @@ -471,20 +630,77 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil continue; } - // Download and save file - log::debug!("Downloading file {} to {:?}", file.id, file_path); + // Check if we've already downloaded this file content (by hash) in another album + // If so, we can copy it instead of downloading again + let need_download = if let Some(hash) = content_hash + && let Some(existing_path) = exported_hashes.get(hash) + && existing_path != &file_path + { + log::info!( + "File {} has same content as {}, copying instead of downloading", + file.id, + existing_path.display() + ); - // Ensure directory exists - if let Some(parent) = file_path.parent() { - fs::create_dir_all(parent).await?; - } + // Copy the existing file to the new location + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent).await?; + } + fs::copy(existing_path, &file_path).await?; - // Download encrypted file - let encrypted_data = api.download_file(&account.email, file.id).await?; + // For live photos, also copy the MOV component + let is_live_photo = metadata + .as_ref() + .map(|m| m.is_live_photo()) + .unwrap_or(false); - // The file nonce/header is stored separately in the API response - let file_nonce = - match base64::engine::general_purpose::STANDARD.decode(&file.file.decryption_header) { + if is_live_photo { + // Try to copy MOV file + let existing_mov = existing_path.with_extension("MOV"); + let new_mov = file_path.with_extension("MOV"); + if existing_mov.exists() { + log::debug!( + "Copying live photo MOV component from {:?} to {:?}", + existing_mov, + new_mov + ); + fs::copy(&existing_mov, &new_mov).await.ok(); + } else { + // Try lowercase .mov + let existing_mov_lower = existing_path.with_extension("mov"); + let new_mov_lower = file_path.with_extension("mov"); + if existing_mov_lower.exists() { + log::debug!( + "Copying live photo mov component from {:?} to {:?}", + existing_mov_lower, + new_mov_lower + ); + fs::copy(&existing_mov_lower, &new_mov_lower).await.ok(); + } + } + } + + false + } else { + true + }; + + // Download and save file only if needed + if need_download { + log::debug!("Downloading file {} to {:?}", file.id, file_path); + + // Ensure directory exists + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent).await?; + } + + // Download encrypted file + let encrypted_data = api.download_file(&account.email, file.id).await?; + + // The file nonce/header is stored separately in the API response + let file_nonce = match base64::engine::general_purpose::STANDARD + .decode(&file.file.decryption_header) + { Ok(nonce) => nonce, Err(e) => { log::error!("Failed to decode file nonce for file {}: {}", file.id, e); @@ -492,41 +708,42 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil } }; - // Decrypt the file data using streaming XChaCha20-Poly1305 - // Use chunked decryption for large files - let decrypted = match decrypt_file_data(&encrypted_data, &file_nonce, &file_key) { - Ok(data) => data, - Err(e) => { - log::error!("Failed to decrypt file {}: {}", file.id, e); - log::debug!( - "File size: {}, header length: {}", - encrypted_data.len(), - file_nonce.len() - ); - continue; - } - }; + // Decrypt the file data using streaming XChaCha20-Poly1305 + // Use chunked decryption for large files + let decrypted = match decrypt_file_data(&encrypted_data, &file_nonce, &file_key) { + Ok(data) => data, + Err(e) => { + log::error!("Failed to decrypt file {}: {}", file.id, e); + log::debug!( + "File size: {}, header length: {}", + encrypted_data.len(), + file_nonce.len() + ); + continue; + } + }; - // Check if this is a live photo that needs extraction - let is_live_photo = metadata - .as_ref() - .map(|m| m.is_live_photo()) - .unwrap_or(false); + // Check if this is a live photo that needs extraction + let is_live_photo = metadata + .as_ref() + .map(|m| m.is_live_photo()) + .unwrap_or(false); - if is_live_photo { - // Extract live photo components from ZIP - if let Err(e) = extract_live_photo(&decrypted, &file_path).await { - log::error!("Failed to extract live photo {}: {}", file.id, e); - // Fall back to saving as ZIP + if is_live_photo { + // Extract live photo components from ZIP + if let Err(e) = extract_live_photo(&decrypted, &file_path).await { + log::error!("Failed to extract live photo {}: {}", file.id, e); + // Fall back to saving as ZIP + let mut file_handle = fs::File::create(&file_path).await?; + file_handle.write_all(&decrypted).await?; + file_handle.sync_all().await?; + } + } else { + // Write regular file let mut file_handle = fs::File::create(&file_path).await?; file_handle.write_all(&decrypted).await?; file_handle.sync_all().await?; } - } else { - // Write regular file - let mut file_handle = fs::File::create(&file_path).await?; - file_handle.write_all(&decrypted).await?; - file_handle.sync_all().await?; } exported_files += 1; @@ -536,15 +753,6 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil exported_hashes.insert(hash.clone(), file_path.clone()); } - // Determine album folder for metadata - let album_folder = if let Some(ref name) = collection.name - && !name.is_empty() - { - sanitize_album_name(name) - } else { - "Uncategorized".to_string() - }; - // Write album metadata if not already written for this album if !albums_with_metadata.contains_key(&album_folder) { write_album_metadata(export_path, &album_folder, collection, account.user_id).await?; @@ -559,7 +767,7 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil .file_name() .and_then(|n| n.to_str()) .unwrap_or("file"); - write_file_metadata( + let meta_path = write_file_metadata( export_path, &album_folder, &file, @@ -569,6 +777,16 @@ async fn export_account(storage: &Storage, account: &Account, filter: &ExportFil ) .await?; + // Track the newly exported file + let existing_files = album_existing_files.get_mut(&album_folder).unwrap(); + existing_files.insert( + file.id, + ExistingFile { + file_path: file_path.clone(), + meta_path, + }, + ); + *file_index += 1; // Progress indicator - show every 10 files for better UX @@ -938,7 +1156,7 @@ async fn write_file_metadata( metadata: Option<&FileMetadata>, filename: &str, file_index: usize, -) -> Result<()> { +) -> Result { let meta_dir = export_path.join(album_folder).join(".meta"); fs::create_dir_all(&meta_dir).await?; @@ -960,7 +1178,7 @@ async fn write_file_metadata( let meta_path = meta_dir.join(meta_filename); let json = serde_json::to_string_pretty(&disk_metadata)?; - fs::write(meta_path, json).await?; + fs::write(&meta_path, json).await?; - Ok(()) + Ok(meta_path) } diff --git a/rust/src/models/export_metadata.rs b/rust/src/models/export_metadata.rs index 3e7f994a08..2847d41730 100644 --- a/rust/src/models/export_metadata.rs +++ b/rust/src/models/export_metadata.rs @@ -1,5 +1,5 @@ -use chrono::{Local, TimeZone}; -use serde::{Deserialize, Serialize, Serializer}; +use chrono::{DateTime, Local, TimeZone}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Custom serializer for timestamp fields to match Go CLI's ISO 8601 format fn serialize_timestamp_as_iso8601( @@ -23,6 +23,31 @@ where serializer.serialize_str(&iso_string) } +/// Custom deserializer that can handle both i64 and ISO string timestamps +fn deserialize_timestamp_from_iso_or_i64<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + use serde::de::Error; + + #[derive(Deserialize)] + #[serde(untagged)] + enum TimestampFormat { + Microseconds(i64), + IsoString(String), + } + + match TimestampFormat::deserialize(deserializer)? { + TimestampFormat::Microseconds(micros) => Ok(micros), + TimestampFormat::IsoString(s) => { + // Parse ISO string back to microseconds + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.timestamp_micros()) + .map_err(|e| Error::custom(format!("Invalid ISO timestamp: {}", e))) + } + } +} + /// Album metadata matching Go's export.AlbumMetadata structure #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -62,9 +87,15 @@ pub struct DiskFileMetadata { pub description: Option, #[serde(skip_serializing_if = "Option::is_none")] pub location: Option, - #[serde(serialize_with = "serialize_timestamp_as_iso8601")] + #[serde( + serialize_with = "serialize_timestamp_as_iso8601", + deserialize_with = "deserialize_timestamp_from_iso_or_i64" + )] pub creation_time: i64, // Unix timestamp in microseconds (serialized as ISO 8601) - #[serde(serialize_with = "serialize_timestamp_as_iso8601")] + #[serde( + serialize_with = "serialize_timestamp_as_iso8601", + deserialize_with = "deserialize_timestamp_from_iso_or_i64" + )] pub modification_time: i64, // Unix timestamp in microseconds (serialized as ISO 8601) pub info: FileInfo, /// Meta filename on disk (excluded from JSON)