diff --git a/Cargo.lock b/Cargo.lock index 52a749eccc..11fa2e8c75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -887,6 +887,7 @@ dependencies = [ "rpassword", "serde", "serde_json", + "serial_test", "sha256", "tempfile", "thiserror 1.0.69", diff --git a/ant-cli/Cargo.toml b/ant-cli/Cargo.toml index c7247837db..9da220a0c2 100644 --- a/ant-cli/Cargo.toml +++ b/ant-cli/Cargo.toml @@ -58,6 +58,7 @@ criterion = "0.5.1" eyre = "0.6.8" rand = { version = "~0.8.5", features = ["small_rng"] } rayon = "1.8.0" +serial_test = "3.0" tempfile = "3.6.0" [lints] diff --git a/ant-cli/src/access/cached_payments.rs b/ant-cli/src/access/cached_payments.rs index 5aa67da21e..e1692e2d80 100644 --- a/ant-cli/src/access/cached_payments.rs +++ b/ant-cli/src/access/cached_payments.rs @@ -18,7 +18,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; const PAYMENT_EXPIRATION_SECS: u64 = 3600 * 24 * 30; pub fn get_payments_dir() -> Result { - let dir = super::data_dir::get_client_data_dir_path()?; + let dir = super::data_dir::get_client_data_dir_base()?; let payments_dir = dir.join("payments"); std::fs::create_dir_all(&payments_dir) .wrap_err("Could not create cached payments directory")?; diff --git a/ant-cli/src/access/data_dir.rs b/ant-cli/src/access/data_dir.rs index 35e6ea35ac..81d465147b 100644 --- a/ant-cli/src/access/data_dir.rs +++ b/ant-cli/src/access/data_dir.rs @@ -6,23 +6,126 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. +use autonomi::Wallet; use color_eyre::{ Section, eyre::{Context, Result, eyre}, }; use std::path::PathBuf; +use thiserror::Error; -pub fn get_client_data_dir_path() -> Result { +#[derive(Debug, Clone, Error)] +pub enum DataDirError { + #[error( + "Multiple accounts found: {0:?}. Please specify which account to use or provide the SECRET_KEY for the account you want to use" + )] + MultipleAccounts(Vec), + #[error( + "No existing user data directories found. Please provide the SECRET_KEY for the account you want to use" + )] + NoExistingUserDirFound, +} + +/// Get the client data directory path. This is the directory that contains the user data for ALL users. +pub fn get_client_data_dir_base() -> Result { let mut home_dirs = dirs_next::data_dir() .ok_or_else(|| eyre!("Failed to obtain data dir, your OS might not be supported."))?; home_dirs.push("autonomi"); home_dirs.push("client"); - std::fs::create_dir_all(home_dirs.as_path()) + Ok(home_dirs) +} + +/// Get the client data directory path. This is the directory that contains the user data for a SINGLE user. +/// For the general data directory case, use [`get_client_data_dir_base`] instead. +/// Automatically detects the wallet directory to use: +/// - if the SECRET_KEY is available, uses it to get the wallet address +/// - if only one user directory exists, uses it +/// - if multiple user directories exist, returns error +/// - if no user directories exist, returns error +pub fn get_client_user_data_dir() -> Result { + let base_dir = get_client_data_dir_base()?; + + // Get the wallet address to use + let wallet_addr = match get_wallet_pk() { + Ok(pk) => pk, + Err(_) => get_wallet_addr_from_existing_user_dirs()?, + }; + + // Migrate legacy data if needed (user data stored directly under client/ without wallet address) + super::data_dir_migration::migrate_legacy_data_if_needed(&wallet_addr)?; + + // Create the wallet directory + let mut wallet_dir = base_dir; + wallet_dir.push(&wallet_addr); + std::fs::create_dir_all(wallet_dir.as_path()) .wrap_err("Failed to create data dir") .with_suggestion(|| { format!( - "make sure you have the correct permissions to access the data dir: {home_dirs:?}" + "make sure you have the correct permissions to access the data dir: {wallet_dir:?}" ) })?; - Ok(home_dirs) + + Ok(wallet_dir) +} + +fn get_wallet_addr_from_existing_user_dirs() -> Result { + // Check if there are any existing accounts user data directories + let existing_users = get_existing_user_dirs()?; + + match &existing_users[..] { + // Exactly one account exists, use it + [one] => Ok(one.clone()), + // No accounts exist yet, try to get address from current environment + // First try from SECRET_KEY env var + [] => Err(DataDirError::NoExistingUserDirFound.into()), + // Multiple wallets exist, try SECRET_KEY env var else return error + [_, ..] => Err(DataDirError::MultipleAccounts(existing_users).into()), + } +} + +/// Get existing wallet directories under the client data dir +fn get_existing_user_dirs() -> Result> { + let base_dir = get_client_data_dir_base()?; + + if !base_dir.exists() { + return Ok(Vec::new()); + } + + let mut wallet_dirs = Vec::new(); + + if let Ok(entries) = std::fs::read_dir(&base_dir) { + for entry in entries.flatten() { + if entry.path().is_dir() { + let dir_name = entry.file_name().to_string_lossy().to_string(); + // Check if it looks like a wallet address (starts with 0x and has the right length) + if dir_name.starts_with("0x") && dir_name.len() == 42 { + wallet_dirs.push(dir_name); + } + } + } + } + + Ok(wallet_dirs) +} + +/// Get all existing account data directory paths +/// Returns a vector of (wallet_address, path) tuples +pub fn get_all_client_data_dir_paths() -> Result> { + let base_dir = get_client_data_dir_base()?; + let existing_users = get_existing_user_dirs()?; + + let mut paths = Vec::new(); + for user in existing_users { + let path = base_dir.join(&user); + paths.push((user, path)); + } + + Ok(paths) +} + +fn get_wallet_pk() -> Result { + let secret_key = crate::wallet::load_wallet_private_key()?; + let wallet = Wallet::new_from_private_key(crate::wallet::DUMMY_NETWORK, &secret_key) + .map_err(|_| eyre!("Invalid SECRET_KEY provided"))?; + Ok(wallet.address().to_string()) } diff --git a/ant-cli/src/access/data_dir_migration.rs b/ant-cli/src/access/data_dir_migration.rs new file mode 100644 index 0000000000..5a3777e553 --- /dev/null +++ b/ant-cli/src/access/data_dir_migration.rs @@ -0,0 +1,670 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use color_eyre::eyre::{Context, Result, eyre}; +use std::fs; +use std::path::PathBuf; + +// List of items to migrate +const ITEMS_TO_MIGRATE: [&str; 4] = [ + "user_data", + "register_signing_key", + "scratchpad_signing_key", + "pointer_signing_key", +]; + +/// Check if legacy data exists (data directly under client/ without wallet address) +fn legacy_data_exists() -> Result { + let base_dir = super::data_dir::get_client_data_dir_base()?; + Ok(ITEMS_TO_MIGRATE + .iter() + .any(|item| base_dir.join(item).exists())) +} + +/// Migrate legacy user data from the old structure to the new wallet-based structure +/// This should only be called when we have the wallet address available +pub fn migrate_legacy_data_if_needed(wallet_address: &str) -> Result<()> { + if !legacy_data_exists()? { + // No legacy data to migrate + return Ok(()); + } + + let base_dir = super::data_dir::get_client_data_dir_base()?; + + println!( + "Detected legacy user data. Updating your local data file location to the new directory structure with multiple account support..." + ); + + let new_wallet_dir = base_dir.join(wallet_address); + + // Create the new wallet directory if it doesn't exist + fs::create_dir_all(&new_wallet_dir) + .wrap_err("Failed to create wallet directory for migration")?; + + let mut migration_errors = Vec::new(); + + for item in ITEMS_TO_MIGRATE { + let old_path = base_dir.join(item); + let new_path = new_wallet_dir.join(item); + + if old_path.exists() { + // Check if the destination already exists + if new_path.exists() { + if data_is_identical(&old_path, &new_path)? { + println!( + "Skipping migration of {item} as identical data already exists in the new location" + ); + } else { + println!( + "Skipping migration of {item} as different data already exists in the new location (preserving existing data)" + ); + } + continue; + } + + // Perform the migration by moving the directory/file + match fs::rename(&old_path, &new_path) { + Ok(()) => { + println!("Successfully migrated: {item}"); + } + Err(e) => { + // If rename fails (e.g., across filesystems), try manual move + match move_item_fallback(&old_path, &new_path) { + Ok(_) => { + println!("Successfully migrated: {item} (using fallback method)"); + } + Err(fallback_err) => { + migration_errors.push(format!( + "Failed to migrate {item}: {fallback_err} (rename error: {e})" + )); + } + } + } + } + } + } + + if !migration_errors.is_empty() { + return Err(eyre!( + "Migration completed with errors: {:?}", + migration_errors + )); + } + + println!("Migration completed successfully!"); + Ok(()) +} + +/// Fallback move method when fs::rename fails (e.g., across filesystems) +/// This recursively moves files/directories by copying then deleting +fn move_item_fallback(source: &PathBuf, destination: &PathBuf) -> Result<()> { + if source.is_dir() { + move_dir_all(source, destination)?; + } else { + // For files, copy then delete + fs::copy(source, destination).wrap_err("Failed to copy file")?; + fs::remove_file(source).wrap_err("Failed to remove source file after copying")?; + } + + Ok(()) +} + +/// Recursively move a directory (copy then delete) +fn move_dir_all(src: &PathBuf, dst: &PathBuf) -> Result<()> { + fs::create_dir_all(dst)?; + + for entry in fs::read_dir(src)? { + let entry = entry?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + + if src_path.is_dir() { + move_dir_all(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path)?; + fs::remove_file(&src_path)?; + } + } + + // Remove the now-empty source directory + fs::remove_dir(src)?; + + Ok(()) +} + +/// Check if two files or directories contain identical data +fn data_is_identical(path1: &PathBuf, path2: &PathBuf) -> Result { + if path1.is_file() && path2.is_file() { + // Compare file contents + let content1 = fs::read(path1)?; + let content2 = fs::read(path2)?; + Ok(content1 == content2) + } else if path1.is_dir() && path2.is_dir() { + // Recursively compare directories + compare_directories(path1, path2) + } else { + // One is file, one is directory - definitely different + Ok(false) + } +} + +/// Recursively compare two directories to see if they contain identical data +fn compare_directories(dir1: &PathBuf, dir2: &PathBuf) -> Result { + let mut entries1: Vec<_> = fs::read_dir(dir1)? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name()) + .collect(); + + let mut entries2: Vec<_> = fs::read_dir(dir2)? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name()) + .collect(); + + // Sort for consistent comparison + entries1.sort(); + entries2.sort(); + + // If different number of entries or different names, they're different + if entries1 != entries2 { + return Ok(false); + } + + // Recursively check each entry + for entry_name in entries1 { + let path1 = dir1.join(&entry_name); + let path2 = dir2.join(&entry_name); + + if !data_is_identical(&path1, &path2)? { + return Ok(false); + } + } + + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + use tempfile::TempDir; + + fn cleanup_test_data() { + if let Ok(client_dir) = crate::access::data_dir::get_client_data_dir_base() { + let _ = fs::remove_dir_all(&client_dir); + } + } + + /// Check if the current wallet has already been migrated + fn is_wallet_migrated(wallet_address: &str) -> Result { + let base_dir = crate::access::data_dir::get_client_data_dir_base()?; + + let wallet_dir = base_dir.join(wallet_address); + + // Consider it migrated if the wallet directory exists and has some expected content + if wallet_dir.exists() { + let user_data_path = wallet_dir.join("user_data"); + // If the wallet dir exists and has user_data or any signing keys, it's been set up + Ok(user_data_path.exists() + || wallet_dir.join("register_signing_key").exists() + || wallet_dir.join("scratchpad_signing_key").exists() + || wallet_dir.join("pointer_signing_key").exists()) + } else { + Ok(false) + } + } + + fn create_test_file(path: &PathBuf, content: &str) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, content).unwrap(); + } + + fn create_test_dir(path: &PathBuf) { + fs::create_dir_all(path).unwrap(); + } + + fn create_legacy_structure() { + // Use the same function the real code uses + let client_dir = crate::access::data_dir::get_client_data_dir_base() + .expect("Failed to get client data dir base in test"); + + fs::create_dir_all(&client_dir).unwrap(); + + // Create legacy user_data structure + let user_data_dir = client_dir.join("user_data"); + fs::create_dir_all(user_data_dir.join("registers")).unwrap(); + fs::create_dir_all(user_data_dir.join("file_archives")).unwrap(); + fs::create_dir_all(user_data_dir.join("scratchpads")).unwrap(); + fs::create_dir_all(user_data_dir.join("pointers")).unwrap(); + + // Create test files + fs::write( + user_data_dir.join("registers").join("test_register"), + "register_content", + ) + .unwrap(); + fs::write(client_dir.join("register_signing_key"), "test_register_key").unwrap(); + fs::write( + client_dir.join("scratchpad_signing_key"), + "test_scratchpad_key", + ) + .unwrap(); + fs::write(client_dir.join("pointer_signing_key"), "test_pointer_key").unwrap(); + } + + #[test] + #[serial] + fn test_legacy_data_exists() { + cleanup_test_data(); + + // Initially no legacy data + assert!(!legacy_data_exists().unwrap()); + + // Create legacy structure + create_legacy_structure(); + + // Now legacy data should exist + assert!(legacy_data_exists().unwrap()); + } + + #[test] + #[serial] + fn test_migration() { + cleanup_test_data(); + + // Create legacy structure + create_legacy_structure(); + + let wallet_address = "0xMigrationTest1234567890123456789012345678"; + + // Verify legacy data exists + assert!(legacy_data_exists().unwrap()); + + // Perform migration + migrate_legacy_data_if_needed(wallet_address).unwrap(); + + // Check that files were moved to new location + let client_dir = crate::access::data_dir::get_client_data_dir_base() + .expect("Failed to get client data dir base in test"); + let wallet_dir = client_dir.join(wallet_address); + + // New structure should exist + assert!(wallet_dir.join("user_data").exists()); + assert!(wallet_dir.join("user_data").join("registers").exists()); + assert!(wallet_dir.join("register_signing_key").exists()); + assert!(wallet_dir.join("scratchpad_signing_key").exists()); + assert!(wallet_dir.join("pointer_signing_key").exists()); + + // Old structure should be removed + assert!(!client_dir.join("user_data").exists()); + assert!(!client_dir.join("register_signing_key").exists()); + assert!(!client_dir.join("scratchpad_signing_key").exists()); + assert!(!client_dir.join("pointer_signing_key").exists()); + + // Legacy data should no longer exist + assert!(!legacy_data_exists().unwrap()); + } + + #[test] + #[serial] + fn test_migration_skip_when_destination_exists() { + cleanup_test_data(); + + // Create legacy structure + create_legacy_structure(); + + let wallet_address = "0xSkipDestinationTest567890123456789012345"; + let client_dir = crate::access::data_dir::get_client_data_dir_base() + .expect("Failed to get client data dir base in test"); + let wallet_dir = client_dir.join(wallet_address); + + // Create destination with different content + fs::create_dir_all(&wallet_dir).unwrap(); + fs::write(wallet_dir.join("register_signing_key"), "existing_key").unwrap(); + + // Perform migration + migrate_legacy_data_if_needed(wallet_address).unwrap(); + + // Existing file should not be overwritten + let content = fs::read_to_string(wallet_dir.join("register_signing_key")).unwrap(); + assert_eq!(content, "existing_key"); + + // Legacy file should still exist since we skipped it + assert!(client_dir.join("register_signing_key").exists()); + } + + #[test] + #[serial] + fn test_no_migration_when_no_legacy_data() { + cleanup_test_data(); + + let wallet_address = "0xNoMigrationTest567890123456789012345678"; + + // No legacy data to migrate + assert!(!legacy_data_exists().unwrap()); + + // Migration should succeed without doing anything + migrate_legacy_data_if_needed(wallet_address).unwrap(); + + // Migration shouldn't create anything since there's no legacy data + assert!(!legacy_data_exists().unwrap()); + } + + #[test] + #[serial] + fn test_is_wallet_migrated() { + cleanup_test_data(); + + let wallet_address = "0xWalletMigratedTest567890123456789012345678"; + + // Initially not migrated + assert!(!is_wallet_migrated(wallet_address).unwrap()); + + // Create wallet directory with user_data + let client_dir = crate::access::data_dir::get_client_data_dir_base() + .expect("Failed to get client data dir base in test"); + let wallet_dir = client_dir.join(wallet_address); + fs::create_dir_all(wallet_dir.join("user_data")).unwrap(); + + // Now should be considered migrated + assert!(is_wallet_migrated(wallet_address).unwrap()); + } + + // === Data Comparison Tests === + + #[test] + fn test_data_is_identical_files() { + let temp_dir = TempDir::new().unwrap(); + let base = temp_dir.path(); + + // Test identical files + let file1 = base.join("file1.txt"); + let file2 = base.join("file2.txt"); + create_test_file(&file1, "Hello world"); + create_test_file(&file2, "Hello world"); + assert!(data_is_identical(&file1, &file2).unwrap()); + + // Test different files + let file3 = base.join("file3.txt"); + create_test_file(&file3, "Different content"); + assert!(!data_is_identical(&file1, &file3).unwrap()); + + // Test empty files + let empty1 = base.join("empty1.txt"); + let empty2 = base.join("empty2.txt"); + create_test_file(&empty1, ""); + create_test_file(&empty2, ""); + assert!(data_is_identical(&empty1, &empty2).unwrap()); + + // Test binary content + let bin1 = base.join("bin1.dat"); + let bin2 = base.join("bin2.dat"); + fs::write(&bin1, [0u8, 1u8, 255u8]).unwrap(); + fs::write(&bin2, [0u8, 1u8, 255u8]).unwrap(); + assert!(data_is_identical(&bin1, &bin2).unwrap()); + } + + #[test] + fn test_data_is_identical_directories() { + let temp_dir = TempDir::new().unwrap(); + let base = temp_dir.path(); + + // Create identical directory structures + let dir1 = base.join("dir1"); + let dir2 = base.join("dir2"); + + // Populate dir1 + create_test_file(&dir1.join("file1.txt"), "content1"); + create_test_file(&dir1.join("subdir/file2.txt"), "content2"); + create_test_file(&dir1.join("subdir/file3.txt"), "content3"); + create_test_dir(&dir1.join("empty_dir")); + + // Populate dir2 with identical content + create_test_file(&dir2.join("file1.txt"), "content1"); + create_test_file(&dir2.join("subdir/file2.txt"), "content2"); + create_test_file(&dir2.join("subdir/file3.txt"), "content3"); + create_test_dir(&dir2.join("empty_dir")); + + assert!(data_is_identical(&dir1, &dir2).unwrap()); + + // Test different directories - different file content + let dir3 = base.join("dir3"); + create_test_file(&dir3.join("file1.txt"), "different content"); + create_test_file(&dir3.join("subdir/file2.txt"), "content2"); + create_test_file(&dir3.join("subdir/file3.txt"), "content3"); + create_test_dir(&dir3.join("empty_dir")); + + assert!(!data_is_identical(&dir1, &dir3).unwrap()); + + // Test different directories - missing file + let dir4 = base.join("dir4"); + create_test_file(&dir4.join("file1.txt"), "content1"); + create_test_file(&dir4.join("subdir/file2.txt"), "content2"); + // Missing file3.txt + create_test_dir(&dir4.join("empty_dir")); + + assert!(!data_is_identical(&dir1, &dir4).unwrap()); + + // Test different directories - extra file + let dir5 = base.join("dir5"); + create_test_file(&dir5.join("file1.txt"), "content1"); + create_test_file(&dir5.join("subdir/file2.txt"), "content2"); + create_test_file(&dir5.join("subdir/file3.txt"), "content3"); + create_test_file(&dir5.join("extra_file.txt"), "extra"); + create_test_dir(&dir5.join("empty_dir")); + + assert!(!data_is_identical(&dir1, &dir5).unwrap()); + } + + #[test] + fn test_data_is_identical_mixed_types() { + let temp_dir = TempDir::new().unwrap(); + let base = temp_dir.path(); + + let file = base.join("test_file.txt"); + let dir = base.join("test_dir"); + + create_test_file(&file, "content"); + create_test_dir(&dir); + + // File vs directory should be different + assert!(!data_is_identical(&file, &dir).unwrap()); + assert!(!data_is_identical(&dir, &file).unwrap()); + } + + #[test] + fn test_data_is_identical_empty_directories() { + let temp_dir = TempDir::new().unwrap(); + let base = temp_dir.path(); + + let empty_dir1 = base.join("empty1"); + let empty_dir2 = base.join("empty2"); + + create_test_dir(&empty_dir1); + create_test_dir(&empty_dir2); + + assert!(data_is_identical(&empty_dir1, &empty_dir2).unwrap()); + } + + // === Enhanced Migration Tests === + + #[test] + #[serial] + fn test_migration_with_identical_data_skips_and_cleans_up() { + cleanup_test_data(); + + // Create legacy structure + create_legacy_structure(); + + let wallet_address = "0xIdenticalDataTest567890123456789012345678"; + let client_dir = crate::access::data_dir::get_client_data_dir_base() + .expect("Failed to get client data dir base in test"); + let wallet_dir = client_dir.join(wallet_address); + + // Create destination with IDENTICAL content + fs::create_dir_all(&wallet_dir).unwrap(); + create_test_file( + &wallet_dir.join("register_signing_key"), + "test_register_key", + ); + create_test_file( + &wallet_dir.join("scratchpad_signing_key"), + "test_scratchpad_key", + ); + create_test_file(&wallet_dir.join("pointer_signing_key"), "test_pointer_key"); + + // Create identical user_data structure + let dest_user_data = wallet_dir.join("user_data"); + create_test_dir(&dest_user_data.join("registers")); + create_test_dir(&dest_user_data.join("file_archives")); + create_test_dir(&dest_user_data.join("scratchpads")); + create_test_dir(&dest_user_data.join("pointers")); + create_test_file( + &dest_user_data.join("registers").join("test_register"), + "register_content", + ); + + // Perform migration + migrate_legacy_data_if_needed(wallet_address).unwrap(); + + // Destination should still exist with original content + assert!(wallet_dir.join("register_signing_key").exists()); + let content = fs::read_to_string(wallet_dir.join("register_signing_key")).unwrap(); + assert_eq!(content, "test_register_key"); + + // Legacy files should still exist since we skipped migration (they were identical) + // This is the correct behavior - we don't delete originals when we skip migration + assert!(client_dir.join("register_signing_key").exists()); + assert!(client_dir.join("user_data").exists()); + } + + #[test] + #[serial] + fn test_migration_with_nested_directory_structure() { + cleanup_test_data(); + + let client_dir = crate::access::data_dir::get_client_data_dir_base().unwrap(); + + // Create complex nested legacy structure + let user_data_dir = client_dir.join("user_data"); + create_test_file(&user_data_dir.join("registers").join("reg1"), "register1"); + create_test_file(&user_data_dir.join("registers").join("reg2"), "register2"); + create_test_file( + &user_data_dir + .join("file_archives") + .join("deep") + .join("nested") + .join("file.txt"), + "nested content", + ); + create_test_file( + &user_data_dir.join("scratchpads").join("scratch1"), + "scratch data", + ); + create_test_dir(&user_data_dir.join("pointers").join("empty_subdir")); + + let wallet_address = "0xNested567890123456789012345678901234567890"; + + // Verify legacy data exists + assert!(legacy_data_exists().unwrap()); + + // Perform migration + migrate_legacy_data_if_needed(wallet_address).unwrap(); + + // Check nested structure was preserved + let wallet_dir = client_dir.join(wallet_address); + assert!( + wallet_dir + .join("user_data") + .join("registers") + .join("reg1") + .exists() + ); + assert!( + wallet_dir + .join("user_data") + .join("registers") + .join("reg2") + .exists() + ); + assert!( + wallet_dir + .join("user_data") + .join("file_archives") + .join("deep") + .join("nested") + .join("file.txt") + .exists() + ); + + // Check content integrity + let content = fs::read_to_string( + wallet_dir + .join("user_data") + .join("file_archives") + .join("deep") + .join("nested") + .join("file.txt"), + ) + .unwrap(); + assert_eq!(content, "nested content"); + + // Check empty directory was created + assert!( + wallet_dir + .join("user_data") + .join("pointers") + .join("empty_subdir") + .exists() + ); + assert!( + wallet_dir + .join("user_data") + .join("pointers") + .join("empty_subdir") + .is_dir() + ); + } + + #[test] + #[serial] + fn test_migration_partial_failure_reporting() { + // This test is more complex as it requires mocking filesystem failures + // For now, we'll test the error aggregation logic with a simpler case + cleanup_test_data(); + + let client_dir = crate::access::data_dir::get_client_data_dir_base().unwrap(); + let wallet_address = "0xFailTest567890123456789012345678901234567890"; + let wallet_dir = client_dir.join(wallet_address); + + // Create legacy structure + create_test_file( + &client_dir.join("register_signing_key"), + "test_register_key", + ); + + // Create destination directory but make migration destination already exist with different content + fs::create_dir_all(&wallet_dir).unwrap(); + create_test_file( + &wallet_dir.join("register_signing_key"), + "different_existing_key", + ); + + // This should succeed without error since we handle existing different files gracefully + let result = migrate_legacy_data_if_needed(wallet_address); + assert!(result.is_ok()); + + // The existing file should be preserved (not overwritten) + let content = fs::read_to_string(wallet_dir.join("register_signing_key")).unwrap(); + assert_eq!(content, "different_existing_key"); + + // Legacy file should still exist since it wasn't moved + assert!(client_dir.join("register_signing_key").exists()); + } +} diff --git a/ant-cli/src/access/keys.rs b/ant-cli/src/access/keys.rs index 8250a5743d..d7c74c2714 100644 --- a/ant-cli/src/access/keys.rs +++ b/ant-cli/src/access/keys.rs @@ -50,7 +50,7 @@ pub fn get_vault_secret_key() -> Result { } pub fn create_register_signing_key_file(key: RegisterSecretKey) -> Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory to write key to")?; let file_path = dir.join(REGISTER_SIGNING_KEY_FILE); fs::write(&file_path, key.to_hex()).wrap_err("Could not write key to file")?; @@ -76,7 +76,7 @@ pub fn get_register_signing_key() -> Result { }; // try from data dir - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err(format!("Failed to obtain register signing key from env var: {why_env_failed}, reading from disk also failed as couldn't access data dir")) .with_suggestion(|| format!("make sure you've provided the {REGISTER_SIGNING_KEY_ENV} env var")) .with_suggestion(|| "you can generate a new secret key with the `register generate-key` subcommand")?; @@ -93,7 +93,7 @@ pub fn get_register_signing_key() -> Result { } pub fn get_register_signing_key_path() -> Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory for register signing key")?; let file_path = dir.join(REGISTER_SIGNING_KEY_FILE); Ok(file_path) @@ -102,7 +102,7 @@ pub fn get_register_signing_key_path() -> Result { // --------- Scratchpad keys ---------- pub fn get_scratchpad_signing_key_path() -> Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory for scratchpad signing key")?; let file_path = dir.join(SCRATCHPAD_SIGNING_KEY_FILE); Ok(file_path) @@ -116,7 +116,7 @@ pub fn get_scratchpad_general_signing_key() -> Result { }; // try from data dir - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err(format!("Failed to obtain scratchpad signing key from env var: {why_env_failed}, reading from disk also failed as couldn't access data dir")) .with_suggestion(|| format!("make sure you've provided the {SCRATCHPAD_SIGNING_KEY_ENV} env var")) .with_suggestion(|| "you can generate a new secret key with the `scratchpad generate-key` subcommand")?; @@ -153,7 +153,7 @@ pub fn parse_scratchpad_signing_key(key_hex: &str) -> Result Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory to write key to")?; let file_path = dir.join(SCRATCHPAD_SIGNING_KEY_FILE); fs::write(&file_path, key.to_hex()).wrap_err("Could not write key to file")?; @@ -163,7 +163,7 @@ pub fn create_scratchpad_signing_key_file(key: ScratchpadSecretKey) -> Result Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory for pointer signing key")?; let file_path = dir.join(POINTER_SIGNING_KEY_FILE); Ok(file_path) @@ -177,7 +177,7 @@ pub fn get_pointer_general_signing_key() -> Result { }; // try from data dir - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err(format!("Failed to obtain pointer signing key from env var: {why_env_failed}, reading from disk also failed as couldn't access data dir")) .with_suggestion(|| format!("make sure you've provided the {POINTER_SIGNING_KEY_ENV} env var")) .with_suggestion(|| "you can generate a new secret key with the `pointer generate-key` subcommand")?; @@ -214,7 +214,7 @@ pub fn parse_pointer_signing_key(key_hex: &str) -> Result { } pub fn create_pointer_signing_key_file(key: ScratchpadSecretKey) -> Result { - let dir = super::data_dir::get_client_data_dir_path() + let dir = super::data_dir::get_client_user_data_dir() .wrap_err("Could not access directory to write key to")?; let file_path = dir.join(POINTER_SIGNING_KEY_FILE); fs::write(&file_path, key.to_hex()).wrap_err("Could not write key to file")?; diff --git a/ant-cli/src/access/mod.rs b/ant-cli/src/access/mod.rs index d0fa21f77e..e47a14ed13 100644 --- a/ant-cli/src/access/mod.rs +++ b/ant-cli/src/access/mod.rs @@ -8,5 +8,6 @@ pub mod cached_payments; pub mod data_dir; +pub mod data_dir_migration; pub mod keys; pub mod user_data; diff --git a/ant-cli/src/access/user_data.rs b/ant-cli/src/access/user_data.rs index 956ec61277..66217cde5d 100644 --- a/ant-cli/src/access/user_data.rs +++ b/ant-cli/src/access/user_data.rs @@ -21,7 +21,7 @@ use autonomi::{ use color_eyre::eyre::Context; use color_eyre::eyre::Result; -use super::data_dir::get_client_data_dir_path; +use super::data_dir::{get_all_client_data_dir_paths, get_client_user_data_dir}; use serde::{Deserialize, Serialize}; @@ -72,8 +72,9 @@ pub fn get_local_user_data() -> Result { Ok(user_data) } -pub fn get_local_private_file_archives() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_private_file_archives_from_path( + data_dir: &std::path::Path, +) -> Result> { let user_data_path = data_dir.join("user_data"); let private_file_archives_path = user_data_path.join("private_file_archives"); std::fs::create_dir_all(&private_file_archives_path)?; @@ -93,8 +94,13 @@ pub fn get_local_private_file_archives() -> Result Result> { + let data_dir = get_client_user_data_dir()?; + get_private_file_archives_from_path(&data_dir) +} + pub fn get_local_private_archive_access(local_addr: &str) -> Result { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let private_file_archives_path = user_data_path.join("private_file_archives"); let file_path = private_file_archives_path.join(local_addr); @@ -106,7 +112,7 @@ pub fn get_local_private_archive_access(local_addr: &str) -> Result Result { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let private_files_path = user_data_path.join("private_files"); let file_path = private_files_path.join(local_addr); @@ -116,8 +122,7 @@ pub fn get_local_private_file_access(local_addr: &str) -> Result { Ok(private_file_access) } -pub fn get_local_registers() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_registers_from_path(data_dir: &std::path::Path) -> Result> { let user_data_path = data_dir.join("user_data"); let registers_path = user_data_path.join("registers"); std::fs::create_dir_all(®isters_path)?; @@ -137,8 +142,13 @@ pub fn get_local_registers() -> Result> { Ok(registers) } +pub fn get_local_registers() -> Result> { + let data_dir = get_client_user_data_dir()?; + get_registers_from_path(&data_dir) +} + pub fn get_name_of_local_register_with_address(address: &RegisterAddress) -> Result { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let registers_path = user_data_path.join("registers"); let file_path = registers_path.join(address.to_hex()); @@ -146,8 +156,9 @@ pub fn get_name_of_local_register_with_address(address: &RegisterAddress) -> Res Ok(file_content) } -pub fn get_local_public_file_archives() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_public_file_archives_from_path( + data_dir: &std::path::Path, +) -> Result> { let user_data_path = data_dir.join("user_data"); let file_archives_path = user_data_path.join("file_archives"); std::fs::create_dir_all(&file_archives_path)?; @@ -166,6 +177,11 @@ pub fn get_local_public_file_archives() -> Result Result> { + let data_dir = get_client_user_data_dir()?; + get_public_file_archives_from_path(&data_dir) +} + pub fn write_local_user_data(user_data: &UserData) -> Result<()> { let UserData { file_archives, @@ -226,7 +242,7 @@ pub fn write_local_user_data(user_data: &UserData) -> Result<()> { } pub fn write_local_register(register: &RegisterAddress, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let registers_path = user_data_path.join("registers"); std::fs::create_dir_all(®isters_path)?; @@ -235,7 +251,7 @@ pub fn write_local_register(register: &RegisterAddress, name: &str) -> Result<() } pub fn write_local_public_file_archive(archive: String, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let file_archives_path = user_data_path.join("file_archives"); std::fs::create_dir_all(&file_archives_path)?; @@ -248,7 +264,7 @@ pub fn write_local_private_file_archive( local_addr: String, name: &str, ) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let private_file_archives_path = user_data_path.join("private_file_archives"); std::fs::create_dir_all(&private_file_archives_path)?; @@ -262,7 +278,7 @@ pub fn write_local_private_file_archive( } pub fn write_local_private_file(datamap_hex: String, local_addr: String, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let private_files_path = user_data_path.join("private_files"); std::fs::create_dir_all(&private_files_path)?; @@ -275,8 +291,9 @@ pub fn write_local_private_file(datamap_hex: String, local_addr: String, name: & Ok(()) } -pub fn get_local_private_files() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_private_files_from_path( + data_dir: &std::path::Path, +) -> Result> { let user_data_path = data_dir.join("user_data"); let private_files_path = user_data_path.join("private_files"); let mut files = HashMap::new(); @@ -307,8 +324,13 @@ pub fn get_local_private_files() -> Result Result> { + let data_dir = get_client_user_data_dir()?; + get_private_files_from_path(&data_dir) +} + pub fn write_local_public_file(data_address: String, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let public_files_path = user_data_path.join("public_files"); std::fs::create_dir_all(&public_files_path)?; @@ -321,8 +343,7 @@ pub fn write_local_public_file(data_address: String, name: &str) -> Result<()> { Ok(()) } -pub fn get_local_public_files() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_public_files_from_path(data_dir: &std::path::Path) -> Result> { let user_data_path = data_dir.join("user_data"); let public_files_path = user_data_path.join("public_files"); let mut files = HashMap::new(); @@ -343,8 +364,13 @@ pub fn get_local_public_files() -> Result> { Ok(files) } +pub fn get_local_public_files() -> Result> { + let data_dir = get_client_user_data_dir()?; + get_public_files_from_path(&data_dir) +} + pub fn write_local_scratchpad(address: ScratchpadAddress, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let scratchpads_path = user_data_path.join("scratchpads"); std::fs::create_dir_all(&scratchpads_path)?; @@ -352,8 +378,7 @@ pub fn write_local_scratchpad(address: ScratchpadAddress, name: &str) -> Result< Ok(()) } -pub fn get_local_scratchpads() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_scratchpads_from_path(data_dir: &std::path::Path) -> Result> { let user_data_path = data_dir.join("user_data"); let scratchpads_path = user_data_path.join("scratchpads"); std::fs::create_dir_all(&scratchpads_path)?; @@ -371,8 +396,13 @@ pub fn get_local_scratchpads() -> Result> { Ok(scratchpads) } +pub fn get_local_scratchpads() -> Result> { + let data_dir = get_client_user_data_dir()?; + get_scratchpads_from_path(&data_dir) +} + pub fn write_local_pointer(address: PointerAddress, name: &str) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let pointers_path = user_data_path.join("pointers"); std::fs::create_dir_all(&pointers_path)?; @@ -380,8 +410,7 @@ pub fn write_local_pointer(address: PointerAddress, name: &str) -> Result<()> { Ok(()) } -pub fn get_local_pointers() -> Result> { - let data_dir = get_client_data_dir_path()?; +fn get_pointers_from_path(data_dir: &std::path::Path) -> Result> { let user_data_path = data_dir.join("user_data"); let pointers_path = user_data_path.join("pointers"); std::fs::create_dir_all(&pointers_path)?; @@ -399,9 +428,14 @@ pub fn get_local_pointers() -> Result> { Ok(pointers) } +pub fn get_local_pointers() -> Result> { + let data_dir = get_client_user_data_dir()?; + get_pointers_from_path(&data_dir) +} + /// Write a pointer value to local user data for caching pub fn write_local_pointer_value(name: &str, pointer: &Pointer) -> Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let pointer_values_path = user_data_path.join("pointer_values"); std::fs::create_dir_all(&pointer_values_path)?; @@ -414,7 +448,7 @@ pub fn write_local_pointer_value(name: &str, pointer: &Pointer) -> Result<()> { /// Get cached pointer value from local storage pub fn get_local_pointer_value(name: &str) -> Result { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let pointer_values_path = user_data_path.join("pointer_values"); std::fs::create_dir_all(&pointer_values_path)?; @@ -427,7 +461,7 @@ pub fn get_local_pointer_value(name: &str) -> Result { /// Get cached pointer values from local storage pub fn get_local_pointer_values() -> Result> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let pointer_values_path = user_data_path.join("pointer_values"); std::fs::create_dir_all(&pointer_values_path)?; @@ -450,7 +484,7 @@ pub fn get_local_pointer_values() -> Result Result<()> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let scratchpad_values_path = user_data_path.join("scratchpad_values"); std::fs::create_dir_all(&scratchpad_values_path)?; @@ -463,7 +497,7 @@ pub fn write_local_scratchpad_value(name: &str, scratchpad: &Scratchpad) -> Resu /// Get cached scratchpad value from local storage pub fn get_local_scratchpad_value(name: &str) -> Result { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let scratchpad_values_path = user_data_path.join("scratchpad_values"); std::fs::create_dir_all(&scratchpad_values_path)?; @@ -476,7 +510,7 @@ pub fn get_local_scratchpad_value(name: &str) -> Result { /// Get cached scratchpad values from local storage pub fn get_local_scratchpad_values() -> Result> { - let data_dir = get_client_data_dir_path()?; + let data_dir = get_client_user_data_dir()?; let user_data_path = data_dir.join("user_data"); let scratchpad_values_path = user_data_path.join("scratchpad_values"); std::fs::create_dir_all(&scratchpad_values_path)?; @@ -496,3 +530,99 @@ pub fn get_local_scratchpad_values() -> Result Result)>> { + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let registers = get_registers_from_path(&path)?; + results.push((account, registers)); + } + + Ok(results) +} + +/// Get all private file archives from all accounts +pub fn get_all_local_private_file_archives() +-> Result)>> { + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let archives = get_private_file_archives_from_path(&path)?; + results.push((account, archives)); + } + + Ok(results) +} + +/// Get all public file archives from all accounts +pub fn get_all_local_public_file_archives() -> Result)>> +{ + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let archives = get_public_file_archives_from_path(&path)?; + results.push((account, archives)); + } + + Ok(results) +} + +/// Get all scratchpads from all accounts +pub fn get_all_local_scratchpads() -> Result)>> { + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let scratchpads = get_scratchpads_from_path(&path)?; + results.push((account, scratchpads)); + } + + Ok(results) +} + +/// Get all pointers from all accounts +pub fn get_all_local_pointers() -> Result)>> { + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let pointers = get_pointers_from_path(&path)?; + results.push((account, pointers)); + } + + Ok(results) +} + +/// Get all private files from all accounts +pub fn get_all_local_private_files() -> Result)>> +{ + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let files = get_private_files_from_path(&path)?; + results.push((account, files)); + } + + Ok(results) +} + +/// Get all public files from all accounts +pub fn get_all_local_public_files() -> Result)>> { + let accounts_data = get_all_client_data_dir_paths()?; + let mut results = Vec::new(); + + for (account, path) in accounts_data { + let files = get_public_files_from_path(&path)?; + results.push((account, files)); + } + + Ok(results) +} diff --git a/ant-cli/src/wallet/fs.rs b/ant-cli/src/wallet/fs.rs index 718d775b85..c8f6a02ebf 100644 --- a/ant-cli/src/wallet/fs.rs +++ b/ant-cli/src/wallet/fs.rs @@ -25,7 +25,7 @@ pub static SELECTED_WALLET_ADDRESS: OnceLock = OnceLock::new(); /// Creates the wallets folder if it is missing and returns the folder path. pub(crate) fn get_client_wallet_dir_path() -> Result { - let mut home_dirs = crate::access::data_dir::get_client_data_dir_path() + let mut home_dirs = crate::access::data_dir::get_client_data_dir_base() .wrap_err("Failed to get wallet directory")?; home_dirs.push("wallets"); diff --git a/ant-protocol/build.rs b/ant-protocol/build.rs index 024ec0b305..47432b32bf 100644 --- a/ant-protocol/build.rs +++ b/ant-protocol/build.rs @@ -8,7 +8,10 @@ fn main() -> Result<(), Box> { tonic_build::configure() - .format(false) // Disable rustfmt formatting - .compile(&["./src/antnode_proto/antnode.proto"], &["./src/antnode_proto"])?; + .format(false) // Disable rustfmt formatting + .compile( + &["./src/antnode_proto/antnode.proto"], + &["./src/antnode_proto"], + )?; Ok(()) } diff --git a/autonomi/src/networking/driver/swarm_events.rs b/autonomi/src/networking/driver/swarm_events.rs index 74fea386fa..89273b8206 100644 --- a/autonomi/src/networking/driver/swarm_events.rs +++ b/autonomi/src/networking/driver/swarm_events.rs @@ -189,7 +189,11 @@ impl NetworkDriver { // Unrecoganized req/rsp DM indicates peer is in an incorrect version // For such case, it shall be counted as a failure. // Using ranodom id as place holder. - self.pending_tasks.terminate_query(request_id, PeerId::random(), OutboundFailure::UnsupportedProtocols)?; + self.pending_tasks.terminate_query( + request_id, + PeerId::random(), + OutboundFailure::UnsupportedProtocols, + )?; } }