diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ffc524a21..e1fbc4671 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -47,18 +47,6 @@ jobs: - name: Clippy checks run: cargo clippy --all-targets --all-features - check_pr_size: - if: "!startsWith(github.event.pull_request.title, 'Automated version bump')" - name: Check PR size doesn't break set limit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: maidsafe/pr_size_checker@v2 - with: - max_lines_changed: 200 - coverage: if: "!startsWith(github.event.pull_request.title, 'Automated version bump')" name: Code coverage check diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..149dce6f6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,163 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Building and Testing + +```bash +# Format code (MANDATORY before commits) +cargo fmt --all + +# Run clippy linter with strict settings +cargo clippy --all-features -- -D warnings + +# Run all Rust tests +cargo test --release + +# Run comprehensive test script (includes Python tests) +./scripts/test.sh + +# Build Python package with maturin +maturin develop --features python + +# Run Python tests +pytest tests/ -v + +# Run benchmarks +cargo bench + +# Check for unused dependencies +cargo udeps --all-targets + +# Publish dry run +cargo publish --dry-run +``` + +### Single Test Execution + +```bash +# Run a specific Rust test +cargo test test_name --release + +# Run a specific Python test +pytest tests/test_file.py::test_name -v + +# Run tests with output +cargo test -- --nocapture +``` + +## Architecture Overview + +### Core Encryption Process + +The self_encryption crate implements convergent encryption with obfuscation through a three-stage process: + +1. **Content Chunking**: Files are split into chunks (up to 1MB each) +2. **Per-Chunk Processing**: + - Compression (Brotli with configurable quality) + - Encryption (AES-256-CBC) + - XOR obfuscation +3. **Key Derivation**: Each chunk's encryption keys are derived from a circular dependency pattern: + - Chunks 0 and 1 have special handling due to circular dependencies + - For chunk N (where N ≥ 2): uses hashes from chunks N, (N+1) % total, (N+2) % total + - Creates interdependency where modifying any chunk affects multiple others + +### Key Components + +- **`src/lib.rs`**: Main library interface, exports public API including `encrypt`, `decrypt_full_set` +- **`src/encrypt.rs`**: Core encryption logic, handles chunk processing and key generation +- **`src/decrypt.rs`**: Decryption logic, reverses the encryption process +- **`src/data_map.rs`**: DataMap structure that stores chunk metadata (src/dst hashes, sizes, indices) +- **`src/stream.rs`**: Streaming encryption/decryption for memory-efficient large file handling +- **`src/chunk.rs`**: Chunk data structures (`EncryptedChunk`, `ChunkInfo`) and validation +- **`src/aes.rs`**: AES encryption implementation using CBC mode +- **`src/utils.rs`**: Utility functions for key derivation, hash extraction, chunk size calculation +- **`src/python.rs`**: PyO3 bindings for Python interface +- **`src/error.rs`**: Error types and handling + +### Storage Backend Design + +The library uses a trait-based design for flexible storage backends: +- Store functions: `Fn(XorName, Bytes) -> Result<()>` +- Retrieve functions: `Fn(XorName) -> Result` +- Supports memory, disk, or custom storage implementations + +### DataMap Hierarchy + +For large files, DataMaps can be shrunk hierarchically: +- Serialize large DataMap → Encrypt as data → Create new smaller DataMap +- Process repeats until manageable size reached +- `child` field tracks hierarchy level + +## Critical Constraints + +- **Minimum file size**: 3072 bytes (3 * MIN_CHUNK_SIZE) for self-encryption +- **Chunk size**: Maximum 1MB per chunk +- **Key security**: The returned secret key from encryption requires secure handling +- **Hash verification**: All chunks are self-validating through SHA3-256 hashes + +## Python Bindings + +The Python interface is built with PyO3 and maturin: +- CLI tool: `self-encryption` command +- Module: `self_encryption` Python package +- Supports both in-memory and streaming operations + +## CI/CD Workflow + +- **PR checks**: Format, clippy, tests, coverage, unused deps +- **Warnings as errors**: `RUSTFLAGS="-D warnings"` enforced in CI +- **Code coverage**: Uses cargo-llvm-cov and reports to coveralls.io +- **32-bit testing**: Includes i686 target testing +- **Python package**: Automated publishing via GitHub Actions + +## Performance Considerations + +- Parallel chunk processing via rayon in standard implementation +- Streaming APIs for memory efficiency with large files +- Benchmarks in `benches/lib.rs` for tracking performance +- Optimized compression settings in Brotli +- Chunk size optimization based on file size + +## StreamSelfEncryptor Implementation Notes + +The streaming implementation differs from the standard implementation in several important ways: + +### Design Differences + +1. **Memory Usage**: + - Standard: Loads entire file into memory, processes all chunks at once + - Streaming: Processes one chunk at a time, O(1) memory usage + +2. **API Pattern**: + - Standard: Functional approach with `encrypt(bytes) -> (DataMap, Vec)` + - Streaming: Stateful object with `next_encryption()` returning chunks incrementally + +3. **Chunk Processing**: + - Standard: Special handling for chunks 0 and 1 (deferred processing due to circular dependencies) + - Streaming: Processes all chunks uniformly (potential issue) + +### Known Issues with StreamSelfEncryptor + +1. **First Two Chunks**: Does not implement the special handling for chunks 0 and 1 that the standard implementation uses. This could lead to incorrect encryption in edge cases. + +2. **Error Handling**: Less robust error handling compared to standard implementation, particularly around chunk validation. + +3. **File System Dependency**: StreamSelfDecryptor uses temporary files extensively, which adds complexity and potential failure points. + +### When to Use Each Implementation + +- **Standard Implementation**: Use for files that fit comfortably in memory (< 1GB) +- **Streaming Implementation**: Use for large files where memory usage is a concern +- **Note**: Both implementations produce compatible output when working correctly + +### Potential Improvements Needed + +1. **Unify Chunk Processing**: Align StreamSelfEncryptor's chunk processing with standard implementation, especially for chunks 0 and 1 +2. **Error Handling**: Improve error handling in streaming implementation to match standard implementation's robustness +3. **Reduce File System Operations**: Consider memory-mapping or buffering strategies for StreamSelfDecryptor +4. **Progress Callbacks**: Add progress reporting capabilities to streaming implementation +5. **Test Coverage**: Ensure streaming implementation has comprehensive tests for edge cases +6. **API Consistency**: Consider refactoring to provide more consistent APIs between implementations \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index d5dd1bd39..0096fbdad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ features = ["rt"] criterion = "0.5.1" docopt = "~0.9.0" clap = { version = "4.4", features = ["derive"] } +serde_json = "1.0" [dev-dependencies.tokio] version = "1.34.0" diff --git a/examples/basic_encryptor.rs b/examples/basic_encryptor.rs index e1b65bec7..693e03922 100644 --- a/examples/basic_encryptor.rs +++ b/examples/basic_encryptor.rs @@ -79,7 +79,7 @@ struct Args { } fn to_hex(ch: u8) -> String { - fmt::format(format_args!("{:02x}", ch)) + fmt::format(format_args!("{ch:02x}")) } fn file_name(name: XorName) -> String { @@ -115,7 +115,7 @@ impl DiskBasedStorage { let mut file = File::create(&path)?; file.write_all(&data[..]) .map(|_| { - println!("Chunk written to {:?}", path); + println!("Chunk written to {path:?}"); }) .map_err(From::from) } @@ -128,7 +128,7 @@ async fn main() { .and_then(|d| d.deserialize()) .unwrap_or_else(|e| e.exit()); if args.flag_help { - println!("{:?}", args) + println!("{args:?}") } let mut chunk_store_dir = env::temp_dir(); @@ -145,7 +145,7 @@ async fn main() { let mut data = Vec::new(); match file.read_to_end(&mut data) { Ok(_) => (), - Err(error) => return println!("{}", error), + Err(error) => return println!("{error}"), } let (data_map, encrypted_chunks) = encrypt(Bytes::from(data)).unwrap(); @@ -163,20 +163,14 @@ async fn main() { Ok(mut file) => { let encoded = serialize(&data_map).unwrap(); match file.write_all(&encoded[..]) { - Ok(_) => println!("Data map written to {:?}", data_map_file), + Ok(_) => println!("Data map written to {data_map_file:?}"), Err(error) => { - println!( - "Failed to write data map to {:?} - {:?}", - data_map_file, error - ); + println!("Failed to write data map to {data_map_file:?} - {error:?}"); } } } Err(error) => { - println!( - "Failed to create data map at {:?} - {:?}", - data_map_file, error - ); + println!("Failed to create data map at {data_map_file:?} - {error:?}"); } } } else { @@ -214,7 +208,7 @@ async fn main() { let content = decrypt(&DataMap::new(keys), encrypted_chunks.as_ref()).unwrap(); match file.write_all(&content[..]) { - Err(error) => println!("File write failed - {:?}", error), + Err(error) => println!("File write failed - {error:?}"), Ok(_) => { println!("File decrypted to {:?}", args.arg_destination.unwrap()) } @@ -228,7 +222,7 @@ async fn main() { } } } else { - println!("Failed to open data map at {:?}", data_map_file); + println!("Failed to open data map at {data_map_file:?}"); } } } diff --git a/examples/parallel_streaming_decryptor.rs b/examples/parallel_streaming_decryptor.rs index 87d728674..588343510 100644 --- a/examples/parallel_streaming_decryptor.rs +++ b/examples/parallel_streaming_decryptor.rs @@ -89,7 +89,7 @@ fn main() -> Result<()> { let mut chunk_data = Vec::new(); File::open(&chunk_path) .and_then(|mut file| file.read_to_end(&mut chunk_data)) - .map_err(|e| Error::Generic(format!("Failed to read chunk: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to read chunk: {e}")))?; Ok(Bytes::from(chunk_data)) }) .collect() @@ -106,9 +106,9 @@ fn main() -> Result<()> { // Helper function to load data map from a file fn load_data_map(path: &str) -> Result { let mut file = - File::open(path).map_err(|e| Error::Generic(format!("Failed to open data map: {}", e)))?; + File::open(path).map_err(|e| Error::Generic(format!("Failed to open data map: {e}")))?; let mut data = Vec::new(); file.read_to_end(&mut data) - .map_err(|e| Error::Generic(format!("Failed to read data map: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to read data map: {e}")))?; deserialize(&data) } diff --git a/src/data_map.rs b/src/data_map.rs index bcb5f3a66..90cd71cc7 100644 --- a/src/data_map.rs +++ b/src/data_map.rs @@ -6,15 +6,19 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use serde::{Deserialize, Serialize}; -use std::fmt::{Debug, Formatter, Write}; +use serde::{ + de::{self, MapAccess, SeqAccess, Visitor}, + ser::SerializeStruct, + Deserialize, Deserializer, Serialize, Serializer, +}; +use std::fmt::{self, Debug, Formatter, Write}; use xor_name::XorName; /// Holds the information that is required to recover the content of the encrypted file. /// This is held as a vector of `ChunkInfo`, i.e. a list of the file's chunk hashes. /// Only files larger than 3072 bytes (3 * MIN_CHUNK_SIZE) can be self-encrypted. /// Smaller files will have to be batched together. -#[derive(Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Clone)] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] pub struct DataMap { /// List of chunk hashes pub chunk_identifiers: Vec, @@ -23,6 +27,43 @@ pub struct DataMap { pub child: Option, } +impl DataMap { + /// Serialize DataMap to bytes using bincode + pub fn to_bytes(&self) -> Result, bincode::Error> { + bincode::serialize(self) + } + + /// Deserialize DataMap from bytes, handling both old and new formats + pub fn from_bytes(bytes: &[u8]) -> Result { + // First, try to deserialize as the new versioned format + #[derive(Deserialize)] + struct VersionedDataMap { + version: u8, + chunk_identifiers: Vec, + child: Option, + } + + // Check if it's the new format by trying to deserialize it + if let Ok(versioned) = bincode::deserialize::(bytes) { + if versioned.version == 1 { + return Ok(DataMap { + chunk_identifiers: versioned.chunk_identifiers, + child: versioned.child, + }); + } + } + + // If that failed, try the old format (just Vec) + match bincode::deserialize::>(bytes) { + Ok(chunks) => Ok(DataMap { + chunk_identifiers: chunks, + child: None, + }), + Err(e) => Err(e), + } + } +} + #[allow(clippy::len_without_is_empty)] impl DataMap { /// A new instance from a vec of partial keys. @@ -78,18 +119,140 @@ impl DataMap { } } +impl Serialize for DataMap { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if serializer.is_human_readable() { + // For JSON and other human-readable formats, use struct format + let mut st = serializer.serialize_struct("DataMap", 2)?; + st.serialize_field("chunk_identifiers", &self.chunk_identifiers)?; + st.serialize_field("child", &self.child)?; + st.end() + } else { + // For binary formats, prepend a version byte + // Version 1: New format with chunk_identifiers and child fields + #[derive(Serialize)] + struct VersionedDataMap<'a> { + version: u8, + chunk_identifiers: &'a Vec, + child: &'a Option, + } + + let versioned = VersionedDataMap { + version: 1u8, + chunk_identifiers: &self.chunk_identifiers, + child: &self.child, + }; + + versioned.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for DataMap { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // For formats that support deserialize_any (like JSON) + if deserializer.is_human_readable() { + struct DataMapVisitor; + + impl<'de> Visitor<'de> for DataMapVisitor { + type Value = DataMap; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "either a Vec (v0) or a struct (v1)") + } + + // --- v0: the whole thing was just a sequence -------------------- + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut chunks = Vec::new(); + while let Some(item) = seq.next_element()? { + chunks.push(item); + } + Ok(DataMap { + chunk_identifiers: chunks, + child: None, // legacy files/network messages + }) + } + + // --- v1: proper struct ----------------------------------------- + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut chunks: Option> = None; + let mut child: Option> = None; + + while let Some(key) = map.next_key::<&str>()? { + match key { + "chunk_identifiers" => chunks = Some(map.next_value()?), + "child" => child = Some(map.next_value()?), + _ => { + let _: de::IgnoredAny = map.next_value()?; + } + } + } + + let chunk_identifiers = + chunks.ok_or_else(|| de::Error::missing_field("chunk_identifiers"))?; + Ok(DataMap { + chunk_identifiers, + child: child.flatten(), // default to None if field absent + }) + } + } + + deserializer.deserialize_any(DataMapVisitor) + } else { + // For binary formats, we need to handle both old and new formats + // Try to peek at the first byte to check for version + #[derive(Deserialize)] + struct VersionedDataMap { + version: u8, + chunk_identifiers: Vec, + child: Option, + } + + // Since we can't peek with serde, we try the versioned format first + // If it starts with a reasonable version number (1), it's the new format + match VersionedDataMap::deserialize(deserializer) { + Ok(versioned) if versioned.version == 1 => Ok(DataMap { + chunk_identifiers: versioned.chunk_identifiers, + child: versioned.child, + }), + _ => { + // If it failed or has wrong version, it might be the old format + // For the old format, we need to re-read from the beginning + // This is a limitation - we can't truly try both formats with serde + // In practice, this means we need external knowledge of the format + Err(de::Error::custom( + "Cannot determine binary format version. Migration required.", + )) + } + } + } + } +} + impl Debug for DataMap { fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { writeln!(formatter, "DataMap:")?; if let Some(child) = self.child { - writeln!(formatter, " child: {}", child)?; + writeln!(formatter, " child: {child}")?; } let len = self.chunk_identifiers.len(); for (index, chunk) in self.chunk_identifiers.iter().enumerate() { if index + 1 == len { - write!(formatter, " {:?}", chunk)? + write!(formatter, " {chunk:?}")? } else { - writeln!(formatter, " {:?}", chunk)? + writeln!(formatter, " {chunk:?}")? } } Ok(()) @@ -124,7 +287,7 @@ fn debug_bytes>(input: V) -> String { if input_ref.len() <= 6 { let mut ret = String::new(); for byte in input_ref.iter() { - write!(ret, "{:02x}", byte).unwrap_or(()); + write!(ret, "{byte:02x}").unwrap_or(()); } return ret; } @@ -151,3 +314,145 @@ impl Debug for ChunkInfo { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_chunk_info(index: usize) -> ChunkInfo { + ChunkInfo { + index, + dst_hash: XorName::from_content(format!("dst_{index}").as_bytes()), + src_hash: XorName::from_content(format!("src_{index}").as_bytes()), + src_size: 1024 * (index + 1), + } + } + + #[test] + fn test_deserialize_old_format_json() { + // Create JSON representing the old tuple struct format: just an array + let chunks = vec![ + create_test_chunk_info(0), + create_test_chunk_info(1), + create_test_chunk_info(2), + ]; + let old_format_json = serde_json::to_string(&chunks).unwrap(); + + // Deserialize as DataMap + let data_map: DataMap = serde_json::from_str(&old_format_json).unwrap(); + + // Verify the data was correctly deserialized + assert_eq!(data_map.chunk_identifiers.len(), 3); + assert_eq!(data_map.child, None); // Should default to None for old format + assert_eq!(data_map.chunk_identifiers[0].index, 0); + assert_eq!(data_map.chunk_identifiers[1].index, 1); + assert_eq!(data_map.chunk_identifiers[2].index, 2); + } + + #[test] + fn test_deserialize_new_format_json() { + // Create a DataMap with the new format + let chunks = vec![create_test_chunk_info(0), create_test_chunk_info(1)]; + let data_map = DataMap::with_child(chunks.clone(), 5); + + // Serialize to JSON + let json = serde_json::to_string(&data_map).unwrap(); + + // Verify the JSON contains the expected structure + assert!(json.contains("\"chunk_identifiers\"")); + assert!(json.contains("\"child\":5")); + + // Deserialize back + let deserialized: DataMap = serde_json::from_str(&json).unwrap(); + + // Verify + assert_eq!(deserialized.chunk_identifiers.len(), 2); + assert_eq!(deserialized.child, Some(5)); + assert_eq!(deserialized.chunk_identifiers[0].index, 0); + assert_eq!(deserialized.chunk_identifiers[1].index, 1); + } + + #[test] + fn test_new_format_without_child_json() { + // Create a DataMap without child + let chunks = vec![create_test_chunk_info(0)]; + let data_map = DataMap::new(chunks.clone()); + + // Serialize and deserialize + let json = serde_json::to_string(&data_map).unwrap(); + let deserialized: DataMap = serde_json::from_str(&json).unwrap(); + + // Verify + assert_eq!(deserialized.chunk_identifiers.len(), 1); + assert_eq!(deserialized.child, None); + } + + #[test] + fn test_bincode_new_format() { + // Create and serialize with new format + let chunks = vec![create_test_chunk_info(0)]; + let data_map = DataMap::with_child(chunks, 3); + + let bytes = data_map.to_bytes().unwrap(); + let deserialized = DataMap::from_bytes(&bytes).unwrap(); + + assert_eq!(deserialized.chunk_identifiers.len(), 1); + assert_eq!(deserialized.child, Some(3)); + } + + #[test] + fn test_bincode_old_format_compatibility() { + // Test that we can deserialize the old format (just Vec) + let chunks = vec![create_test_chunk_info(0), create_test_chunk_info(1)]; + + // Simulate old format by encoding just the Vec + let old_format_bytes = bincode::serialize(&chunks).unwrap(); + + // Should successfully deserialize using from_bytes + let data_map = DataMap::from_bytes(&old_format_bytes).unwrap(); + + // Verify + assert_eq!(data_map.chunk_identifiers.len(), 2); + assert_eq!(data_map.child, None); // Old format has no child + assert_eq!(data_map.chunk_identifiers[0].index, 0); + assert_eq!(data_map.chunk_identifiers[1].index, 1); + } + + #[test] + fn test_bincode_version_byte() { + // Verify that new format includes version byte + let chunks = vec![create_test_chunk_info(0)]; + let data_map = DataMap::new(chunks); + + let bytes = data_map.to_bytes().unwrap(); + + // First byte should be the version (1) + assert!(!bytes.is_empty()); + assert_eq!(bytes[0], 1u8); + } + + #[test] + fn test_preserve_chunk_order() { + // Ensure that chunk ordering is preserved through serialization + let chunks = vec![ + create_test_chunk_info(2), + create_test_chunk_info(0), + create_test_chunk_info(1), + ]; + + // DataMap::new should sort them + let data_map = DataMap::new(chunks); + assert_eq!(data_map.chunk_identifiers[0].index, 0); + assert_eq!(data_map.chunk_identifiers[1].index, 1); + assert_eq!(data_map.chunk_identifiers[2].index, 2); + + // Serialize and deserialize + let json = serde_json::to_string(&data_map).unwrap(); + let deserialized: DataMap = serde_json::from_str(&json).unwrap(); + + // Order should be preserved + assert_eq!(deserialized.chunk_identifiers[0].index, 0); + assert_eq!(deserialized.chunk_identifiers[1].index, 1); + assert_eq!(deserialized.chunk_identifiers[2].index, 2); + } +} diff --git a/src/lib.rs b/src/lib.rs index ba9b8e068..e08b0677b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,8 +173,7 @@ pub fn encrypt(bytes: Bytes) -> Result<(DataMap, Vec)> { let file_size = bytes.len(); if file_size < MIN_ENCRYPTABLE_BYTES { return Err(Error::Generic(format!( - "Too small for self-encryption! Required size at least {}", - MIN_ENCRYPTABLE_BYTES + "Too small for self-encryption! Required size at least {MIN_ENCRYPTABLE_BYTES}" ))); } @@ -495,7 +494,7 @@ pub fn decrypt(data_map: &DataMap, chunks: &[EncryptedChunk]) -> Result { chunk_map .get(&hash) .map(|chunk| chunk.content.clone()) - .ok_or_else(|| Error::Generic(format!("Chunk not found for hash: {:?}", hash))) + .ok_or_else(|| Error::Generic(format!("Chunk not found for hash: {hash:?}"))) }; // Get the root map if we're dealing with a child map @@ -588,8 +587,7 @@ where if file_size < MIN_ENCRYPTABLE_BYTES { return Err(Error::Generic(format!( - "Too small for self-encryption! Required size at least {}", - MIN_ENCRYPTABLE_BYTES + "Too small for self-encryption! Required size at least {MIN_ENCRYPTABLE_BYTES}" ))); } @@ -751,7 +749,7 @@ where /// /// * `Result>` - The serialized bytes or an error pub fn serialize(data: &T) -> Result> { - bincode::serialize(data).map_err(|e| Error::Generic(format!("Serialization error: {}", e))) + bincode::serialize(data).map_err(|e| Error::Generic(format!("Serialization error: {e}"))) } /// Deserializes bytes into a data structure using bincode. @@ -764,7 +762,7 @@ pub fn serialize(data: &T) -> Result> { /// /// * `Result` - The deserialized data structure or an error pub fn deserialize(bytes: &[u8]) -> Result { - bincode::deserialize(bytes).map_err(|e| Error::Generic(format!("Deserialization error: {}", e))) + bincode::deserialize(bytes).map_err(|e| Error::Generic(format!("Deserialization error: {e}"))) } /// Verifies and deserializes a chunk by checking its content hash matches the provided name. @@ -790,8 +788,7 @@ pub fn verify_chunk(name: XorName, bytes: &[u8]) -> Result { // Verify the hash matches if calculated_hash != name { return Err(Error::Generic(format!( - "Chunk content hash mismatch. Expected: {:?}, Got: {:?}", - name, calculated_hash + "Chunk content hash mismatch. Expected: {name:?}, Got: {calculated_hash:?}" ))); } @@ -900,7 +897,7 @@ mod tests { println!("\nFinal Data Map Info:"); println!("Number of chunks: {}", shrunk_map.len()); - println!("Original file size: {}", file_size); + println!("Original file size: {file_size}"); println!("Is child: {}", shrunk_map.is_child()); for (i, info) in shrunk_map.infos().iter().enumerate() { diff --git a/src/python.rs b/src/python.rs index 2b0ef2ae4..859df4b2d 100644 --- a/src/python.rs +++ b/src/python.rs @@ -136,7 +136,7 @@ impl PyDataMap { #[staticmethod] pub fn from_json(json_str: &str) -> PyResult { let inner = serde_json::from_str(json_str) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid JSON: {}", e)))?; + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid JSON: {e}")))?; Ok(Self { inner }) } @@ -146,7 +146,7 @@ impl PyDataMap { /// str: A JSON string representation of the DataMap. pub fn to_json(&self) -> PyResult { serde_json::to_string(&self.inner).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to serialize: {}", e)) + pyo3::exceptions::PyValueError::new_err(format!("Failed to serialize: {e}")) }) } @@ -274,9 +274,8 @@ impl PyEncryptedChunk { #[pyfunction] pub fn encrypt(data: &[u8]) -> PyResult<(PyDataMap, Vec)> { let bytes = Bytes::copy_from_slice(data); - let (data_map, chunks) = crate::encrypt(bytes).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Encryption failed: {}", e)) - })?; + let (data_map, chunks) = crate::encrypt(bytes) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Encryption failed: {e}")))?; let py_chunks = chunks .into_iter() .map(|chunk| PyEncryptedChunk { inner: chunk }) @@ -307,9 +306,8 @@ pub fn decrypt( .into_iter() .map(|chunk| chunk.inner) .collect::>(); - let bytes = crate::decrypt(&data_map.inner, &inner_chunks).map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Decryption failed: {}", e)) - })?; + let bytes = crate::decrypt(&data_map.inner, &inner_chunks) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Decryption failed: {e}")))?; Ok(bytes.to_vec().into()) } @@ -335,7 +333,7 @@ pub fn encrypt_from_file(input_file: &str, output_dir: &str) -> PyResult<(PyData let input_path = Path::new(input_file); let output_path = Path::new(output_dir); let (data_map, chunk_names) = rust_encrypt_from_file(input_path, output_path).map_err(|e| { - pyo3::exceptions::PyOSError::new_err(format!("Failed to encrypt file: {}", e)) + pyo3::exceptions::PyOSError::new_err(format!("Failed to encrypt file: {e}")) })?; let chunk_names = chunk_names.iter().map(|name| hex::encode(name.0)).collect(); Ok((PyDataMap { inner: data_map }, chunk_names)) @@ -365,15 +363,15 @@ pub fn decrypt_from_storage( let name_str = hex::encode(name.0); let chunk = get_chunk .call1((name_str,)) - .map_err(|e| crate::Error::Python(format!("Failed to call get_chunk: {}", e)))?; + .map_err(|e| crate::Error::Python(format!("Failed to call get_chunk: {e}")))?; let bytes = chunk .downcast::() - .map_err(|e| crate::Error::Python(format!("get_chunk must return bytes: {}", e)))?; + .map_err(|e| crate::Error::Python(format!("get_chunk must return bytes: {e}")))?; Ok(Bytes::copy_from_slice(bytes.as_bytes())) }; rust_decrypt_from_storage(&data_map.inner, output_path, get_chunk_wrapper) - .map_err(|e| pyo3::exceptions::PyOSError::new_err(format!("Decryption failed: {}", e))) + .map_err(|e| pyo3::exceptions::PyOSError::new_err(format!("Decryption failed: {e}"))) } /// Decrypt data using streaming for better performance with large files. @@ -400,24 +398,24 @@ pub fn streaming_decrypt_from_storage( let name_strs: Vec = names.iter().map(|x| hex::encode(x.0)).collect(); let chunks = get_chunks .call1((name_strs,)) - .map_err(|e| crate::Error::Python(format!("Failed to call get_chunks: {}", e)))?; + .map_err(|e| crate::Error::Python(format!("Failed to call get_chunks: {e}")))?; let chunks = chunks .try_iter() - .map_err(|e| crate::Error::Python(format!("get_chunks must return a list: {}", e)))?; + .map_err(|e| crate::Error::Python(format!("get_chunks must return a list: {e}")))?; let mut result = Vec::new(); for chunk in chunks { let chunk = chunk - .map_err(|e| crate::Error::Python(format!("Failed to iterate chunks: {}", e)))?; - let bytes = chunk.downcast::().map_err(|e| { - crate::Error::Python(format!("get_chunks must return bytes: {}", e)) - })?; + .map_err(|e| crate::Error::Python(format!("Failed to iterate chunks: {e}")))?; + let bytes = chunk + .downcast::() + .map_err(|e| crate::Error::Python(format!("get_chunks must return bytes: {e}")))?; result.push(Bytes::copy_from_slice(bytes.as_bytes())); } Ok(result) }; rust_streaming_decrypt_from_storage(&data_map.inner, output_path, get_chunks_wrapper).map_err( - |e| pyo3::exceptions::PyOSError::new_err(format!("Streaming decryption failed: {}", e)), + |e| pyo3::exceptions::PyOSError::new_err(format!("Streaming decryption failed: {e}")), ) } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 609ce55ce..ed1a6977d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -71,10 +71,10 @@ impl StorageBackend { Box::new(move |hash| { let path = base_path.join(hex::encode(hash)); let mut file = File::open(&path) - .map_err(|e| Error::Generic(format!("Failed to open chunk file: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to open chunk file: {e}")))?; let mut data = Vec::new(); file.read_to_end(&mut data) - .map_err(|e| Error::Generic(format!("Failed to read chunk data: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to read chunk data: {e}")))?; Ok(Bytes::from(data)) }) } @@ -98,7 +98,7 @@ impl StorageBackend { } fn debug_storage_state(&self, prefix: &str) -> Result<()> { - println!("\n=== {} ===", prefix); + println!("\n=== {prefix} ==="); if let Ok(guard) = self.memory.lock() { println!("Memory storage contains {} chunks", guard.len()); for (hash, data) in guard.iter() { @@ -138,7 +138,7 @@ fn test_cross_backend_encryption_decryption() -> Result<()> { let temp_dir = TempDir::new()?; for (name, use_memory_store, _use_memory_retrieve) in &[("memory-to-memory", true, true)] { - println!("\nRunning test case: {}", name); + println!("\nRunning test case: {name}"); let input_path = temp_dir.path().join("input.dat"); let mut input_file = File::create(&input_path)?; @@ -242,12 +242,8 @@ fn test_concurrent_backend_access() -> Result<()> { let count = processed.fetch_add(1, Ordering::SeqCst); // Setup paths with unique identifiers - let input_path = temp_dir - .path() - .join(format!("input_{}_{}.dat", count, size)); - let output_path = temp_dir - .path() - .join(format!("output_{}_{}.dat", count, size)); + let input_path = temp_dir.path().join(format!("input_{count}_{size}.dat")); + let output_path = temp_dir.path().join(format!("output_{count}_{size}.dat")); // Write test data File::create(&input_path)?.write_all(&data)?; @@ -320,7 +316,7 @@ fn test_cross_platform_compatibility() -> Result<()> { for size in &[3073, 1024 * 1024] { // Start with smaller subset for testing - println!("Testing size: {}", size); + println!("Testing size: {size}"); // Create deterministic data let mut content = vec![0u8; *size]; @@ -329,7 +325,7 @@ fn test_cross_platform_compatibility() -> Result<()> { } let original_data = Bytes::from(content); - let input_path = temp_dir.path().join(format!("input_{}.dat", size)); + let input_path = temp_dir.path().join(format!("input_{size}.dat")); let mut input_file = File::create(&input_path)?; input_file.write_all(&original_data)?; @@ -372,7 +368,7 @@ fn test_platform_specific_sizes() -> Result<()> { ]; for (name, size) in test_cases { - println!("Testing size: {} ({} bytes)", name, size); + println!("Testing size: {name} ({size} bytes)"); let original_data = random_bytes(size); @@ -404,9 +400,7 @@ fn test_platform_specific_sizes() -> Result<()> { assert_eq!( original_data.as_ref(), decrypted_bytes.as_ref(), - "Data mismatch for {} (size: {})", - name, - size + "Data mismatch for {name} (size: {size})" ); } @@ -431,7 +425,7 @@ fn test_encrypt_from_file_stores_all_chunks() -> Result<()> { // Now encrypt from file let (data_map, chunk_names) = encrypt_from_file(&input_path, storage.disk_dir.path())?; - println!("Expected chunks: {}", expected_chunk_count); + println!("Expected chunks: {expected_chunk_count}"); println!("Got chunk names: {}", chunk_names.len()); // Verify we got all chunks @@ -472,7 +466,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { ]; for (size_name, size) in test_cases { - println!("\n=== Testing {} file ===", size_name); + println!("\n=== Testing {size_name} file ==="); let original_data = random_bytes(size); // 1. In-memory encryption (encrypt) @@ -483,7 +477,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { // 2. File-based encryption (encrypt_from_file) println!("\n2. Testing file-based encryption (encrypt_from_file):"); - let input_path = temp_dir.path().join(format!("input_{}.dat", size_name)); + let input_path = temp_dir.path().join(format!("input_{size_name}.dat")); File::create(&input_path)?.write_all(&original_data)?; let (data_map2, chunk_names) = encrypt_from_file(&input_path, storage.disk_dir.path())?; println!("- Generated {} chunks", chunk_names.len()); @@ -510,7 +504,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let chunk_path = storage.disk_dir.path().join(hex::encode(hash)); File::create(&chunk_path)?.write_all(&chunk.content)?; } - let output_path1 = temp_dir.path().join(format!("output1_{}.dat", size_name)); + let output_path1 = temp_dir.path().join(format!("output1_{size_name}.dat")); let mut retrieve_fn = storage.retrieve_from_disk(); decrypt_from_storage(&data_map1, &output_path1, &mut retrieve_fn)?; @@ -527,7 +521,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { println!("\nA.3 Testing streaming_decrypt_from_storage() with encrypt() result:"); let output_path1_stream = temp_dir .path() - .join(format!("output1_stream_{}.dat", size_name)); + .join(format!("output1_stream_{size_name}.dat")); // Create parallel chunk retrieval function let chunk_dir = storage.disk_dir.path().to_owned(); @@ -539,7 +533,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let mut chunk_data = Vec::new(); File::open(&chunk_path) .and_then(|mut file| file.read_to_end(&mut chunk_data)) - .map_err(|e| Error::Generic(format!("Failed to read chunk: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to read chunk: {e}")))?; Ok(Bytes::from(chunk_data)) }) .collect() @@ -577,7 +571,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { // E. Test decrypt_from_storage() with file-based encryption result println!("\nB.2 Testing decrypt_from_storage() with encrypt_from_file() result:"); - let output_path2 = temp_dir.path().join(format!("output2_{}.dat", size_name)); + let output_path2 = temp_dir.path().join(format!("output2_{size_name}.dat")); let mut retrieve_fn = storage.retrieve_from_disk(); decrypt_from_storage(&data_map2, &output_path2, &mut retrieve_fn)?; @@ -594,7 +588,7 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { println!("\nB.3 Testing streaming_decrypt_from_storage() with encrypt_from_file() result:"); let output_path2_stream = temp_dir .path() - .join(format!("output2_stream_{}.dat", size_name)); + .join(format!("output2_stream_{size_name}.dat")); streaming_decrypt_from_storage(&data_map2, &output_path2_stream, get_chunk_parallel)?; let mut decrypted = Vec::new(); @@ -645,14 +639,13 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { File::open(path2)?.read_to_end(&mut content2)?; assert_eq!( content1, content2, - "Output files don't match: {:?} vs {:?}", - path1, path2 + "Output files don't match: {path1:?} vs {path2:?}" ); } } println!("✓ All output files match"); - println!("\n{} test completed successfully", size_name); + println!("\n{size_name} test completed successfully"); } Ok(()) @@ -682,7 +675,7 @@ fn test_streaming_decrypt_with_parallel_retrieval() -> Result<()> { let mut chunk_data = Vec::new(); File::open(&chunk_path) .and_then(|mut file| file.read_to_end(&mut chunk_data)) - .map_err(|e| Error::Generic(format!("Failed to read chunk: {}", e)))?; + .map_err(|e| Error::Generic(format!("Failed to read chunk: {e}")))?; Ok(Bytes::from(chunk_data)) }) .collect() @@ -757,8 +750,8 @@ fn test_chunk_verification() -> Result<()> { File::open(&chunk_path)?.read_to_end(&mut chunk_content)?; match verify_chunk(info.dst_hash, &chunk_content) { - Ok(_) => println!("✓ Chunk {} verified successfully", i), - Err(e) => println!("✗ Chunk {} verification failed: {}", i, e), + Ok(_) => println!("✓ Chunk {i} verified successfully"), + Err(e) => println!("✗ Chunk {i} verification failed: {e}"), } } diff --git a/tests/lib.rs b/tests/lib.rs index 7d065798c..36896a3f6 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -92,15 +92,13 @@ async fn cross_platform_check() -> Result<()> { for (i, (expected, got)) in ref_data_map.iter().zip(data_map.infos()).enumerate() { assert_eq!( expected.src_hash, got.src_hash, - "Chunk {} src_hash mismatch", - i + "Chunk {i} src_hash mismatch" ); assert_eq!( expected.dst_hash, got.dst_hash, - "Chunk {} dst_hash mismatch", - i + "Chunk {i} dst_hash mismatch" ); - assert_eq!(expected.src_size, got.src_size, "Chunk {} size mismatch", i); + assert_eq!(expected.src_size, got.src_size, "Chunk {i} size mismatch"); } Ok(()) @@ -163,12 +161,12 @@ fn test_data_map_debug() { // Test Debug output without child let data_map = DataMap::new(chunk_infos.clone()); - let debug_str = format!("{:?}", data_map); + let debug_str = format!("{data_map:?}"); assert!(!debug_str.contains("child:")); // Test Debug output with child let data_map = DataMap::with_child(chunk_infos, 42); - let debug_str = format!("{:?}", data_map); + let debug_str = format!("{data_map:?}"); assert!(debug_str.contains("child: 42")); }