diff --git a/.axes/axes.toml b/.axes/axes.toml index 5f07d5f..01733c4 100644 --- a/.axes/axes.toml +++ b/.axes/axes.toml @@ -3,6 +3,7 @@ version = "0.6.0" description = "Blazing fast parallel caching for Rust. Serializes complex data to a chunked file format to get CPU maximum power on data loads, but as a simple binary loader." [vars] +_cargo_no_features = "" _cargo_all = " " _cargo_features = "" _cargo_targets = "" @@ -10,10 +11,10 @@ _cargo_all_feat_targets = "().expect("Failed to read lazy"); let sum = lazy.child_a.meta + lazy.child_b.meta; black_box(sum); @@ -62,7 +64,7 @@ fn bench_lazy(c: &mut Criterion) { group.bench_function("lazy_partial_load", |b| { b.iter(|| { - let file_handle = Parcode::open(&path).expect("Failed to open file"); + let file_handle = Parcode::open_bytes(buffer.clone()).expect("Failed to open file"); let lazy = file_handle.root::().expect("Failed to read lazy"); let payload = lazy.child_a.payload.load().expect("Failed to load payload"); black_box(payload.len()); diff --git a/benches/map_access.rs b/benches/map_access.rs index 5dab84c..940b231 100644 --- a/benches/map_access.rs +++ b/benches/map_access.rs @@ -2,8 +2,7 @@ use criterion::{Criterion, criterion_group, criterion_main}; use parcode::{Parcode, ParcodeObject}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use tempfile::NamedTempFile; +use std::{collections::HashMap, sync::Arc}; #[derive(Serialize, Deserialize, ParcodeObject)] struct MapContainer { @@ -21,14 +20,14 @@ fn bench_map(c: &mut Criterion) { } let data = MapContainer { opt_map: map }; - let file = NamedTempFile::new().expect("Failed to create temp file"); - Parcode::save(file.path(), &data).expect("Failed to save parcode data"); - let path = file.path().to_owned(); + let mut buffer = Vec::new(); + Parcode::write(&mut buffer, &data).expect("Failed to write parcode data"); + let buffer = Arc::new(buffer); let mut group = c.benchmark_group("Map Random Access"); group.bench_function("optimized_lookup", |b| { - let file_handle = Parcode::open(&path).expect("Failed to open file"); + let file_handle = Parcode::open_bytes(buffer.clone()).expect("Failed to open file"); let lazy = file_handle .root::() .expect("Failed to read lazy"); diff --git a/benches/performance.rs b/benches/performance.rs index 9cd61ef..dba64b1 100644 --- a/benches/performance.rs +++ b/benches/performance.rs @@ -3,10 +3,9 @@ use criterion::{Criterion, Throughput, criterion_group, criterion_main}; use parcode::{Parcode, ParcodeObject}; use serde::{Deserialize, Serialize}; -use std::fs::File; use std::hint::black_box; -use std::io::BufWriter; -use tempfile::NamedTempFile; +use std::io::Cursor; +use std::sync::Arc; #[derive(Clone, Serialize, Deserialize, ParcodeObject, Debug)] struct BenchItem { @@ -45,12 +44,11 @@ fn bench_writers(c: &mut Criterion) { // 1. Baseline: Bincode (Single Threaded) group.bench_function("bincode_serialize", |b| { + let mut buffer = Vec::new(); b.iter(|| { - let file = NamedTempFile::new().expect("Failed to create temp file"); - let mut writer = BufWriter::new(file); bincode::serde::encode_into_std_write( black_box(raw_data), - &mut writer, + &mut Cursor::new(&mut buffer), bincode::config::standard(), ) .expect("Bincode serialization failed"); @@ -59,9 +57,9 @@ fn bench_writers(c: &mut Criterion) { // 2. Parcode group.bench_function("parcode_save", |b| { + let mut buffer = Vec::new(); b.iter(|| { - let file = NamedTempFile::new().expect("Failed to create temp file"); - Parcode::save(file.path(), black_box(&data)).expect("Failed to save parcode data"); + Parcode::write(&mut buffer, black_box(&data)).expect("Failed to save parcode data"); }); }); @@ -76,20 +74,20 @@ fn bench_readers(c: &mut Criterion) { let data = generate_data(item_count); // Setup files - let bincode_file = NamedTempFile::new().expect("Failed to create temp file"); + let mut bincode_buffer = Vec::new(); + bincode::serde::encode_into_std_write( &data.data, - &mut BufWriter::new(&bincode_file), + &mut bincode_buffer, bincode::config::standard(), ) .expect("Bincode serialization failed"); - let bincode_path = bincode_file.path().to_owned(); - let parcode_file = NamedTempFile::new().expect("Failed to create temp file"); - Parcode::save(parcode_file.path(), &data).expect("Failed to save parcode data"); - let parcode_path = parcode_file.path().to_owned(); + let mut parcode_buffer = Vec::new(); + Parcode::write(&mut parcode_buffer, &data).expect("Failed to save parcode data"); + let parcode_buffer = Arc::new(parcode_buffer); - let file_handle = Parcode::open(&parcode_path).expect("Failed to open file"); + let file_handle = Parcode::open_bytes(parcode_buffer.clone()).expect("Failed to open file"); let root = file_handle.root_node().expect("Failed to get root"); println!( "Chunks detected: {}", @@ -101,19 +99,27 @@ fn bench_readers(c: &mut Criterion) { // 1. Bincode: Standard group.bench_function("bincode_read_all", |b| { b.iter(|| { - let file = File::open(&bincode_path).expect("Failed to open file"); let _res: Vec = bincode::serde::decode_from_std_read( - &mut std::io::BufReader::new(file), + &mut Cursor::new(&bincode_buffer), bincode::config::standard(), ) .expect("Bincode deserialization failed"); }); }); - // 2. Parcode: Random Access (Single item) + // 2. Parcode: Parallel full load + group.bench_function("parcode_read_all", |b| { + b.iter(|| { + let _res: BenchCollection = + Parcode::load_bytes(parcode_buffer.clone()).expect("Failed to open file"); + }); + }); + + // 3. Parcode: Random Access (Single item) group.bench_function("parcode_random_access_10", |b| { b.iter(|| { - let file_handle = Parcode::open(&parcode_path).expect("Failed to open file"); + let file_handle = + Parcode::open_bytes(parcode_buffer.clone()).expect("Failed to open file"); let root = file_handle .root::() .expect("Failed to get root"); @@ -124,17 +130,10 @@ fn bench_readers(c: &mut Criterion) { }); }); - // 3. Parcode: Parallel Stitching - group.bench_function("parcode_read_all_parallel", |b| { - b.iter(|| { - let _res: BenchCollection = Parcode::load(&parcode_path).expect("Failed to open file"); - }); - }); - // 4. Parcode: lazy iterator group.bench_function("parcode_read_all_iter", |b| { b.iter(|| { - let file = Parcode::open(&parcode_path).expect("Failed to open file"); + let file = Parcode::open_bytes(parcode_buffer.clone()).expect("Failed to open file"); let data = file .load_lazy::() .expect("Some error was ocurred"); diff --git a/examples/audit.rs b/examples/audit.rs index baf14db..2f48589 100644 --- a/examples/audit.rs +++ b/examples/audit.rs @@ -9,11 +9,10 @@ use parcode::{Parcode, ParcodeObject}; use serde::{Deserialize, Serialize}; use std::alloc::{GlobalAlloc, Layout, System}; use std::collections::HashMap; -use std::fs::File; -use std::io::{BufReader, BufWriter}; +use std::io::Cursor; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; -use tempfile::NamedTempFile; // ============================================================================ // 1. MEMORY PROFILER @@ -94,7 +93,6 @@ struct Region { #[derive(Serialize, Deserialize, Clone, PartialEq, Debug, ParcodeObject)] struct Zone { id: u32, - // Heavy payload #[parcode(chunkable)] terrain_data: Vec, } @@ -176,9 +174,6 @@ fn main() -> parcode::Result<()> { ALLOCATOR.reset(); let world_data = generate_world(); - let path_par = NamedTempFile::new().expect("Failed to create temp file"); - let path_bin = NamedTempFile::new().expect("Failed to create temp file"); - // ------------------------------------------------------------------------ // TEST 1: WRITING (Serialization) // ------------------------------------------------------------------------ @@ -187,59 +182,78 @@ fn main() -> parcode::Result<()> { // Parcode ALLOCATOR.reset(); let t_start = Instant::now(); - Parcode::save(path_par.path(), &world_data)?; + let mut parcode_buffer = Vec::new(); + Parcode::write(&mut parcode_buffer, &world_data)?; let t_par_write = t_start.elapsed(); let mem_par_write = ALLOCATOR.peak(); - let size_par = path_par.path().metadata()?.len(); + let size_par = parcode_buffer.len(); + let parcode_buffer = Arc::new(parcode_buffer); // Bincode ALLOCATOR.reset(); let t_start = Instant::now(); - { - let mut w = BufWriter::new(File::create(path_bin.path())?); - bincode::serde::encode_into_std_write(&world_data, &mut w, bincode::config::standard()) - .expect("Bincode serialization failed"); - } + let mut bincode_buffer = Vec::new(); + bincode::serde::encode_into_std_write( + &world_data, + &mut bincode_buffer, + bincode::config::standard(), + ) + .expect("Bincode serialization failed"); let t_bin_write = t_start.elapsed(); let mem_bin_write = ALLOCATOR.peak(); - let size_bin = path_bin.path().metadata()?.len(); + let size_bin = bincode_buffer.len(); print_metric("Write Time", t_par_write, t_bin_write); print_ram("Write RAM", mem_par_write, mem_bin_write); print_size("Disk Size", size_par, size_bin); // ------------------------------------------------------------------------ - // TEST 2: COLD START (Open File & Parse Metadata) + // TEST 2: Full Read (Read all data from buffer) // ------------------------------------------------------------------------ - println!("\n[TEST 2: COLD START (Ready to Read)]"); + println!("\n[TEST 2: Full Read]"); - // Parcode: Lazy Read (Solo lee Header + Root Chunk) + // Parcode: deserialize EVERYTHING optionally ALLOCATOR.reset(); let t_start = Instant::now(); - let file_handle = Parcode::open(path_par.path())?; - let lazy_world = file_handle.root::()?; + let _full_world: WorldState = Parcode::load_bytes(parcode_buffer.clone())?; let t_par_open = t_start.elapsed(); let mem_par_open = ALLOCATOR.peak(); // Bincode: Must deserialize EVERYTHING to be usable ALLOCATOR.reset(); let t_start = Instant::now(); - let file = File::open(path_bin.path())?; - let mut br = BufReader::new(file); - let _full_world: WorldState = - bincode::serde::decode_from_std_read(&mut br, bincode::config::standard()) - .expect("Bincode deserialization failed"); + let _full_world: WorldState = bincode::serde::decode_from_std_read( + &mut Cursor::new(&bincode_buffer), + bincode::config::standard(), + ) + .expect("Bincode deserialization failed"); let t_bin_open = t_start.elapsed(); let mem_bin_open = ALLOCATOR.peak(); - print_metric("Time to Ready", t_par_open, t_bin_open); // Expect massive win for Parcode + print_metric("Time to Ready", t_par_open, t_bin_open); + print_ram("RAM to Ready", mem_par_open, mem_bin_open); + + // ------------------------------------------------------------------------ + // TEST 3: COLD START (Parse Metadata vs Full Read) + // ------------------------------------------------------------------------ + println!("\n[TEST 3: COLD START (Parse Metadata vs Full Read)]"); + + // Parcode: Lazy Read (Solo lee Header + Root Chunk) + ALLOCATOR.reset(); + let t_start = Instant::now(); + let file_handle = Parcode::open_bytes(parcode_buffer.clone())?; + let lazy_world = file_handle.root::()?; + let t_par_open = t_start.elapsed(); + let mem_par_open = ALLOCATOR.peak(); + + print_metric("Time to Ready", t_par_open, t_bin_open); print_ram("RAM to Ready", mem_par_open, mem_bin_open); // ------------------------------------------------------------------------ - // TEST 3: DEEP SURGICAL FETCH + // TEST 4: DEEP SURGICAL FETCH (Lazy vs Full Read) // Target: World -> Region[5] -> Zone[20] -> TerrainData (First byte) // ------------------------------------------------------------------------ - println!("\n[TEST 3: DEEP SURGICAL FETCH]"); + println!("\n[TEST 4: DEEP SURGICAL FETCH]"); // Parcode: Navigate Graph ALLOCATOR.reset(); @@ -252,19 +266,14 @@ fn main() -> parcode::Result<()> { let mem_par_deep = ALLOCATOR.peak(); black_box(byte); - // Bincode: Already loaded in memory (zero time now, but paid huge upfront cost). - // To be fair, we compare "Time to fetch specific item from cold disk". - // Bincode Time = Open Time (Test 2) + Access Time (0). - let t_bin_deep = t_bin_open; - - print_metric("Fetch Time", t_par_deep, t_bin_deep); + print_metric("Fetch Time", t_par_deep, t_bin_open); print_ram("Fetch RAM", mem_par_deep, mem_bin_open); // ------------------------------------------------------------------------ - // TEST 4: MAP LOOKUP (Random Access) + // TEST 5: MAP LOOKUP (Random Access) // Target: User #88888 // ------------------------------------------------------------------------ - println!("\n[TEST 4: MAP LOOKUP (User #88888)]"); + println!("\n[TEST 5: MAP LOOKUP (User #88888)]"); // Parcode: Hash -> Bucket -> Scan ALLOCATOR.reset(); @@ -274,11 +283,7 @@ fn main() -> parcode::Result<()> { let mem_par_map = ALLOCATOR.peak(); black_box(user); - // Bincode: Memory lookup (Fast) but heavily taxed by initial load. - // Again, comparing Cold Access Time. - let t_bin_map = t_bin_open; - - print_metric("Lookup Time", t_par_map, t_bin_map); + print_metric("Lookup Time", t_par_map, t_bin_open); print_ram("Lookup RAM", mem_par_map, mem_bin_open); Ok(()) @@ -306,7 +311,7 @@ fn print_ram(label: &str, par: usize, bin: usize) { ); } -fn print_size(label: &str, par: u64, bin: u64) { +fn print_size(label: &str, par: usize, bin: usize) { let p_mb = par as f64 / 1048576.0; let b_mb = bin as f64 / 1048576.0; let overhead = ((p_mb / b_mb) - 1.0) * 100.0; diff --git a/src/api.rs b/src/api.rs index ebd1a44..7915529 100644 --- a/src/api.rs +++ b/src/api.rs @@ -48,9 +48,7 @@ //! let my_data = MyType { val: 42 }; //! //! // Enable compression -//! Parcode::builder() -//! .compression(true) -//! .save("data_custom.par", &my_data)?; +//! Parcode::save("data_custom.par", &my_data)?; //! # std::fs::remove_file("data_custom.par")?; //! # Ok::<(), parcode::ParcodeError>(()) //! ``` @@ -176,9 +174,19 @@ impl Parcode { /// This is a convenience wrapper equivalent to `ParcodeInspector::inspect`. /// /// # Example - /// ```rust,ignore - /// let report = Parcode::inspect("data.par")?; + /// ```rust + /// use parcode::{Parcode, ParcodeObject}; + /// use serde::{Serialize,Deserialize}; + /// + /// #[derive(Serialize, Deserialize, ParcodeObject)] + /// struct MyType { val: i32 } + /// + /// let data = MyType { val: 42 }; + /// let serialized = Parcode::serialize(&data)?; + /// + /// let report = Parcode::inspect_bytes(serialized)?; /// println!("{}", report); + /// # Ok::<(), parcode::ParcodeError>(()) /// ``` #[cfg(not(target_arch = "wasm32"))] pub fn inspect>(path: P) -> Result { @@ -217,19 +225,6 @@ impl Parcode { /// # std::fs::remove_file("data_basic.par").unwrap(); /// ``` /// -/// ### With Compression -/// -/// ```rust -/// use parcode::Parcode; -/// -/// let data = vec![1, 2, 3]; -/// Parcode::builder() -/// .compression(true) -/// .save("data_comp.par", &data)?; -/// # std::fs::remove_file("data_comp.par")?; -/// # Ok::<(), parcode::ParcodeError>(()) -/// ``` -/// /// ## Performance Notes /// /// - The builder itself has negligible overhead (it's just a small struct with flags) @@ -269,7 +264,7 @@ impl ParcodeOptions { /// /// ## Examples /// - /// ```rust + /// ```rust,ignore /// use parcode::{Parcode, ParcodeObject}; /// use serde::{Serialize, Deserialize}; /// @@ -341,9 +336,7 @@ impl ParcodeOptions { /// let data = vec![1, 2, 3, 4, 5]; /// /// // Write with compression - /// Parcode::builder() - /// .compression(true) - /// .save("data_write.par", &data)?; + /// Parcode::save("data_write.par", &data)?; /// # std::fs::remove_file("data_write.par")?; /// # Ok::<(), parcode::ParcodeError>(()) /// ``` @@ -476,7 +469,7 @@ impl ParcodeOptions { let registry = crate::compression::CompressorRegistry::new(); #[cfg(feature = "parallel")] - let root_child_ref = crate::executor::execute_graph_sync( + let root_child_ref = crate::executor::execute_graph_serial( &graph, &seq_writer, ®istry, diff --git a/src/executor.rs b/src/executor.rs index 07641b4..216d151 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -4,17 +4,26 @@ //! to be purely reactive: completed children trigger the scheduling of their parents, //! eliminating the need for a central polling loop and reducing latency. +use std::io::Write; + +#[cfg(feature = "parallel")] use std::sync::Mutex; -use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(feature = "parallel")] +use std::sync::atomic::AtomicBool; +#[cfg(feature = "parallel")] +use std::sync::atomic::Ordering; use crate::compression::CompressorRegistry; use crate::error::{ParcodeError, Result}; use crate::format::{ChildRef, MetaByte}; -use crate::graph::{Node, TaskGraph}; +use crate::graph::TaskGraph; use crate::io::SeqWriter; -use std::io::Write; + +#[cfg(feature = "parallel")] +use crate::graph::Node; /// Context shared among all worker threads during parallel execution. +#[cfg(feature = "parallel")] struct ExecutionContext<'a, 'graph, W> where W: Write + Send, @@ -75,7 +84,7 @@ where #[cfg(any(not(feature = "parallel"), target_arch = "wasm32"))] { - execute_graph_serial_fallback(graph, writer, registry, use_compression) + execute_graph_serial(graph, writer, registry, use_compression) } } // ------------ @@ -143,7 +152,7 @@ where } /// Executes the graph synchronously (single-threaded). -pub fn execute_graph_sync<'a, W>( +pub fn execute_graph_serial<'a, W>( graph: &TaskGraph<'a>, writer: &SeqWriter, registry: &CompressorRegistry, diff --git a/src/reader.rs b/src/reader.rs index 82119b1..7acd7ee 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -632,7 +632,8 @@ impl ParcodeFile { let mut file = File::open(path)?; let file_size = file.metadata()?.len(); - let mut buffer = Vec::with_capacity(file_size as usize); + let mut buffer = + Vec::with_capacity(usize::try_from(file_size).expect("File size too large")); file.read_to_end(&mut buffer)?; diff --git a/tests/api_test.rs b/tests/api_test.rs index 3990181..9016e31 100644 --- a/tests/api_test.rs +++ b/tests/api_test.rs @@ -67,7 +67,7 @@ fn test_memory_io() -> parcode::Result<()> { } /// Explicit Synchronous Write (Forced) -/// Validate `Parcode::write_sync`, `execute_graph_sync` +/// Validate `Parcode::write_sync`, `execute_graph_serial` #[test] fn test_explicit_sync_write() -> parcode::Result<()> { let data = create_complex_data(); @@ -122,6 +122,7 @@ fn test_lazy_load_memory() -> parcode::Result<()> { /// Compression /// Validate `ParcodeOptions`, `CompressorRegistry` #[test] +#[cfg(feature = "lz4_flex")] fn test_compression_config() -> parcode::Result<()> { let data = create_complex_data(); let mut buffer_compressed = Vec::new(); diff --git a/tests/map_test.rs b/tests/map_test.rs index f9e7d3a..59ab5ff 100644 --- a/tests/map_test.rs +++ b/tests/map_test.rs @@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tempfile::NamedTempFile; +#[cfg(feature = "lz4_flex")] #[derive(Serialize, Deserialize, ParcodeObject)] struct UserDatabase { id: u32, @@ -12,6 +13,14 @@ struct UserDatabase { users: HashMap, } +#[cfg(not(feature = "lz4_flex"))] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, ParcodeObject)] +struct UserDatabase { + id: u32, + #[parcode(map)] + users: HashMap, +} + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, ParcodeObject)] struct UserProfile { level: u32,