From 8581742a730f22c2abd2b780c728dbaa2aaccb12 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 26 Aug 2025 05:55:03 +0530 Subject: [PATCH] feat(rust): Integrate DownloadManager with sync command - Added local_path column to files table for tracking downloaded files - Implemented get_pending_downloads() to find files without local_path - Integrated DownloadManager into sync command for full file downloads - Added collection key decryption for file downloads - Generate proper export paths with date/album structure - Track successful downloads and update database with local paths - Added migration to add local_path column to existing databases The sync command now supports full file downloads (not just metadata). Files are downloaded to the export directory with proper organization. Co-Authored-By: Claude --- rust/CONVERSION_PLAN.md | 39 +++--- rust/src/commands/sync.rs | 237 +++++++++++++++++++++++++++++++++++-- rust/src/storage/schema.rs | 14 +++ rust/src/storage/sync.rs | 49 ++++++++ rust/src/sync/download.rs | 16 ++- rust/src/sync/mod.rs | 2 +- 6 files changed, 322 insertions(+), 35 deletions(-) diff --git a/rust/CONVERSION_PLAN.md b/rust/CONVERSION_PLAN.md index ae5ed420d5..775d697f3b 100644 --- a/rust/CONVERSION_PLAN.md +++ b/rust/CONVERSION_PLAN.md @@ -44,8 +44,10 @@ The Rust CLI now has a **fully functional export capability** with proper file d ### Account Management (`/rust/src/commands/account.rs`) - ✅ **Account list** - Display all configured accounts -- ✅ **Account add** (partial) - Add account with stored credentials +- ✅ **Account add** - Full SRP authentication implemented - ✅ Store encrypted credentials in SQLite +- ✅ 2FA/OTP support +- ✅ Proper key derivation with Argon2 ### Metadata Handling (`/rust/src/models/metadata.rs`) - ✅ **Metadata decryption and parsing** @@ -55,24 +57,28 @@ The Rust CLI now has a **fully functional export capability** with proper file d ## In Progress 🚧 +### Sync Command (`/rust/src/commands/sync.rs`) +- ✅ **Metadata sync implemented** +- ✅ Fetch collections with pagination +- ✅ Fetch files metadata per collection (matching Go CLI) +- ✅ Store sync state in SQLite +- ✅ Handle deleted files/collections +- ✅ Per-collection incremental sync tracking for metadata +- ⚠️ **File downloads NOT integrated** - only syncs metadata currently +- 📝 TODO: Integrate DownloadManager for actual file downloads + ### File Download Manager -- ⚠️ Basic structure exists but not fully integrated +- ✅ Basic structure implemented (`/rust/src/sync/download.rs`) +- ✅ Download individual files with decryption +- ✅ Parallel download infrastructure +- ⚠️ **NOT integrated with sync command** - Need to implement: - - Parallel downloads with progress tracking - - Integration with sync command + - Integration with sync command for full sync mode + - Progress tracking UI - Resume interrupted downloads ## Remaining Components 📝 -### Sync Command (`/rust/src/commands/sync.rs`) -- ✅ **Full sync command implemented** -- ✅ Fetch collections with pagination -- ✅ Fetch files per collection (matching Go CLI) -- ✅ Store sync state in SQLite -- ✅ Handle deleted files/collections -- ✅ Metadata-only and full sync modes -- ✅ Per-collection incremental sync tracking - ### Database and Storage (`/rust/src/storage/`) - ✅ **Platform-specific config directory** (`~/.config/ente-cli/`) - ✅ Avoid conflicts with Go CLI path @@ -142,14 +148,15 @@ The Rust CLI now has a **fully functional export capability** with proper file d ### Feature Parity Progress - [x] Multi-account support (storage) - [x] Photos export (basic) -- [x] Sync command (collections and files) +- [x] Sync command (metadata only currently) - [x] Album organization - [x] Deduplicated storage - [x] Platform-specific config paths -- [ ] SRP authentication (using stored tokens currently) +- [x] SRP authentication (fully implemented) +- [ ] Full sync with file downloads - [ ] Locker export - [ ] Auth (2FA) export -- [ ] Incremental sync (partial - needs testing) +- [x] Incremental sync (metadata only) - [ ] Export filters (albums, shared, hidden) ### Data Migration diff --git a/rust/src/commands/sync.rs b/rust/src/commands/sync.rs index cbe78acf73..a68a99e92f 100644 --- a/rust/src/commands/sync.rs +++ b/rust/src/commands/sync.rs @@ -1,9 +1,13 @@ use crate::Result; use crate::api::client::ApiClient; -use crate::models::account::Account; +use crate::api::methods::ApiMethods; +use crate::crypto::secret_box_open; +use crate::models::{account::Account, metadata::FileMetadata}; use crate::storage::Storage; -use crate::sync::{SyncEngine, SyncStats}; +use crate::sync::{SyncEngine, SyncStats, download::DownloadManager}; use base64::Engine; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; pub async fn run_sync( account_email: Option, @@ -83,12 +87,17 @@ async fn sync_account( storage.sync().clear_sync_state(account.id)?; } - // Create sync engine (need to create new storage instance for ownership) + // Create sync engine (need to create new instances for ownership) let db_path = storage .db_path() .ok_or_else(|| crate::Error::Generic("Database path not available".into()))?; + + // Create API client for sync engine + let sync_api_client = ApiClient::new(Some(account.endpoint.clone()))?; + sync_api_client.add_token(&account.email, &token); + let sync_storage = Storage::new(db_path)?; - let sync_engine = SyncEngine::new(api_client, sync_storage, account.clone()); + let sync_engine = SyncEngine::new(sync_api_client, sync_storage, account.clone()); // Run sync println!("Fetching collections and files..."); @@ -99,15 +108,74 @@ async fn sync_account( // Download files if not metadata-only if !metadata_only { - println!("\n📥 Downloading files would happen here (not yet implemented)"); - // TODO: Implement file download using DownloadManager - // let pending_files = sync_engine.get_pending_downloads().await?; - // if !pending_files.is_empty() { - // println!("Found {} files to download", pending_files.len()); - // let download_manager = DownloadManager::new(api_client, storage.clone())?; - // // Set collection keys... - // // Download files... - // } + // Get pending downloads + let pending_files = storage.sync().get_pending_downloads(account.id)?; + + if !pending_files.is_empty() { + println!("\n📥 Found {} files to download", pending_files.len()); + + // Get collections to decrypt collection keys + // Need to fetch from API to get the api::models::Collection type with encrypted_key + let api = ApiMethods::new(&api_client); + let api_collections = api.get_collections(&account.email, 0).await?; + + // Decrypt collection keys + let collection_keys = decrypt_collection_keys( + &api_collections, + &secrets.master_key, + &secrets.secret_key, + )?; + + // Create download manager + // Create a new API client for the download manager + let download_api_client = ApiClient::new(Some(account.endpoint.clone()))?; + download_api_client.add_token(&account.email, &token); + + // Create another storage instance for download manager + let download_storage = Storage::new(db_path)?; + let mut download_manager = DownloadManager::new(download_api_client, download_storage)?; + download_manager.set_collection_keys(collection_keys); + + // Determine export directory + let export_dir = if let Some(ref dir) = account.export_dir { + PathBuf::from(dir) + } else { + std::env::current_dir()?.join("ente-export") + }; + + // Prepare download tasks with proper paths + let download_tasks = prepare_download_tasks( + &pending_files, + &export_dir, + &api_collections, + &download_manager, + ) + .await?; + + // Download files + let download_stats = download_manager + .download_files(&account.email, download_tasks) + .await?; + + // Update local paths in database + for (file, path) in &download_stats.successful_downloads { + storage.sync().update_file_local_path( + account.id, + file.id, + path.to_str().unwrap_or(""), + )?; + } + + println!( + "\n✅ Downloaded {} files successfully", + download_stats.successful + ); + if download_stats.failed > 0 { + println!("❌ Failed to download {} files", download_stats.failed); + } + } else { + println!("\n✨ All files are already downloaded"); + } } else { println!("\n📋 Metadata-only sync completed (skipping file downloads)"); } @@ -141,3 +209,146 @@ fn display_sync_stats(stats: &SyncStats) { ); println!("└─────────────────────────────────────┘"); } + +/// Decrypt collection keys for file decryption +fn decrypt_collection_keys( + collections: &[crate::api::models::Collection], + master_key: &[u8], + _secret_key: &[u8], +) -> Result>> { + use base64::engine::general_purpose::STANDARD as BASE64; + + let mut keys = HashMap::new(); + + for collection in collections { + if collection.is_deleted { + continue; + } + + // Decrypt collection key + let encrypted_bytes = BASE64.decode(&collection.encrypted_key)?; + let nonce_bytes = BASE64.decode(&collection.key_decryption_nonce)?; + + match secret_box_open(&encrypted_bytes, &nonce_bytes, master_key) { + Ok(key) => { + keys.insert(collection.id, key); + } + Err(e) => { + log::warn!( + "Failed to decrypt key for collection {}: {}", + collection.id, + e + ); + } + } + } + + Ok(keys) +} + +/// Prepare download tasks with proper file paths +async fn prepare_download_tasks( + files: &[crate::models::file::RemoteFile], + export_dir: &Path, + collections: &[crate::api::models::Collection], + download_manager: &DownloadManager, +) -> Result> { + use crate::crypto::decrypt_stream; + use base64::engine::general_purpose::STANDARD as BASE64; + use chrono::{TimeZone, Utc}; + + let mut tasks = Vec::new(); + + // Create collection lookup map + let collection_map: HashMap = + collections.iter().map(|c| (c.id, c)).collect(); + + for file in files { + // Get collection for this file + let collection = collection_map.get(&file.collection_id); + + // Try to decrypt metadata to get original filename + let metadata = if let Some(col_key) = + download_manager.collection_keys.get(&file.collection_id) + { + // Decrypt file key first + let file_key = { + let key_bytes = BASE64.decode(&file.encrypted_key)?; + let nonce = BASE64.decode(&file.key_decryption_nonce)?; + secret_box_open(&key_bytes, &nonce, col_key)? + }; + + // Then decrypt metadata if available + // Note: file.metadata is a crate::models::file::MetadataInfo, not FileAttributes + if !file.metadata.encrypted_data.is_empty() { + if !file.metadata.decryption_header.is_empty() { + let encrypted_bytes = BASE64.decode(&file.metadata.encrypted_data)?; + let header_bytes = BASE64.decode(&file.metadata.decryption_header)?; + + match decrypt_stream(&encrypted_bytes, &header_bytes, &file_key) { + Ok(decrypted) => serde_json::from_slice::(&decrypted).ok(), + Err(e) => { + log::warn!("Failed to decrypt metadata for file {}: {}", file.id, e); + None + } + } + } else { + None + } + } else { + None + } + } else { + None + }; + + // Generate export path + let mut path = export_dir.to_path_buf(); + + // Add date-based directory structure + let datetime = Utc + .timestamp_micros(file.updated_at) + .single() + .ok_or_else(|| crate::Error::Generic("Invalid timestamp".into()))?; + + let year = datetime.format("%Y").to_string(); + let month = datetime.format("%m-%B").to_string(); + + path.push(year); + path.push(month); + + // Add collection name if available + if let Some(col) = collection { + if let Some(ref name) = col.name { + if !name.is_empty() && name != "Uncategorized" { + let safe_name: String = name + .chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_', + c if c.is_control() => '_', + c => c, + }) + .collect(); + path.push(safe_name.trim()); + } + } + } + + // Use original filename from metadata or generate fallback + let filename = if let Some(ref meta) = metadata { + if let Some(title) = meta.get_title() { + title.to_string() + } else { + format!("file_{}.jpg", file.id) + } + } else { + format!("file_{}.jpg", file.id) + }; + + path.push(filename); + + tasks.push((file.clone(), path)); + } + + Ok(tasks) +} diff --git a/rust/src/storage/schema.rs b/rust/src/storage/schema.rs index 14e014a63e..0668bba592 100644 --- a/rust/src/storage/schema.rs +++ b/rust/src/storage/schema.rs @@ -74,6 +74,7 @@ pub fn create_tables(conn: &Connection) -> Result<()> { file_info TEXT NOT NULL, metadata TEXT NOT NULL, is_deleted INTEGER NOT NULL DEFAULT 0, + local_path TEXT, updated_at INTEGER NOT NULL, FOREIGN KEY (account_id) REFERENCES accounts(id) ON DELETE CASCADE, FOREIGN KEY (collection_id) REFERENCES collections(collection_id), @@ -129,5 +130,18 @@ pub fn create_tables(conn: &Connection) -> Result<()> { [], )?; + // Add migration for local_path column if it doesn't exist + let has_column = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('files') WHERE name='local_path'", + [], + |row| row.get::<_, i32>(0), + ) + .unwrap_or(0); + + if has_column == 0 { + conn.execute("ALTER TABLE files ADD COLUMN local_path TEXT", [])?; + } + Ok(()) } diff --git a/rust/src/storage/sync.rs b/rust/src/storage/sync.rs index 9e554638fc..3db312566c 100644 --- a/rust/src/storage/sync.rs +++ b/rust/src/storage/sync.rs @@ -204,4 +204,53 @@ impl<'a> SyncStore<'a> { )?; Ok(()) } + + /// Get files that need downloading (no local_path) + pub fn get_pending_downloads(&self, account_id: i64) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT file_id, collection_id, encrypted_key, key_decryption_nonce, + file_info, metadata, is_deleted, updated_at + FROM files + WHERE account_id = ?1 AND is_deleted = 0 AND local_path IS NULL + ORDER BY file_id", + )?; + + let files = stmt + .query_map(params![account_id], |row| { + let file_info: String = row.get(4)?; + let metadata: String = row.get(5)?; + + Ok(RemoteFile { + id: row.get(0)?, + collection_id: row.get(1)?, + owner_id: 0, // Will need to fetch from account + encrypted_key: row.get(2)?, + key_decryption_nonce: row.get(3)?, + file: serde_json::from_str(&file_info).unwrap(), + thumbnail: serde_json::from_str("{}").unwrap(), // Placeholder + metadata: serde_json::from_str(&metadata).unwrap(), + is_deleted: row.get::<_, i32>(6)? != 0, + updated_at: row.get(7)?, + }) + })? + .collect::, _>>()?; + + Ok(files) + } + + /// Update file local path after successful download + pub fn update_file_local_path( + &self, + account_id: i64, + file_id: i64, + local_path: &str, + ) -> Result<()> { + self.conn.execute( + "UPDATE files SET local_path = ?3 + WHERE account_id = ?1 AND file_id = ?2", + params![account_id, file_id, local_path], + )?; + + Ok(()) + } } diff --git a/rust/src/sync/download.rs b/rust/src/sync/download.rs index 37e231ade3..96a8201040 100644 --- a/rust/src/sync/download.rs +++ b/rust/src/sync/download.rs @@ -16,7 +16,7 @@ pub struct DownloadManager { #[allow(dead_code)] storage: Storage, temp_dir: PathBuf, - collection_keys: HashMap>, + pub collection_keys: HashMap>, concurrent_downloads: usize, } @@ -111,9 +111,11 @@ impl DownloadManager { let results: Vec<_> = stream::iter(files) .map(|(file, path)| { let account_id = account_id.to_string(); + let file_clone = file.clone(); + let path_clone = path.clone(); async move { let result = self.download_file(&account_id, &file, &path).await; - (file.id, result) + (file_clone, path_clone, result) } }) .buffer_unordered(self.concurrent_downloads) @@ -121,11 +123,14 @@ impl DownloadManager { .await; // Count results - for (_file_id, result) in results { + for (file, path, result) in results { match result { - Ok(_) => stats.successful += 1, + Ok(_) => { + stats.successful += 1; + stats.successful_downloads.push((file, path)); + } Err(e) => { - log::error!("Download failed: {e}"); + log::error!("Download failed for file {}: {}", file.id, e); stats.failed += 1; } } @@ -238,6 +243,7 @@ pub struct DownloadStats { pub successful: usize, pub failed: usize, pub skipped: usize, + pub successful_downloads: Vec<(RemoteFile, PathBuf)>, } impl DownloadStats { diff --git a/rust/src/sync/mod.rs b/rust/src/sync/mod.rs index a5ad25f651..4d99e3f41e 100644 --- a/rust/src/sync/mod.rs +++ b/rust/src/sync/mod.rs @@ -1,4 +1,4 @@ -mod download; +pub mod download; mod engine; mod files;