Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions .axes/axes.toml
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
name = "parcode"
version = "0.5.3"
version = "0.6.0"
description = "Blazing fast parallel caching for Rust. Serializes complex data to a chunked file format to get CPU maximum power on data loads, but as a simple binary loader."

[scripts.test]
[vars]
_cargo_all = "<vars::_cargo_features> <vars::_cargo_targets> <vars::_cargo_all_feat_targets>"
_cargo_features = "<params::features(alias='-f', default=' ', literal)>"
_cargo_targets = "<params::targets(alias='-t', default=' ', literal)>"
_cargo_all_feat_targets = "<params::all(map='--all-features --all-targets', default=' ')>"
GREETING = "Hello from an axes variable!"

[scripts]
fmt = "cargo fmt -- --check"
clippy = "cargo clippy <vars::_cargo_all> <params> -- -D warnings"
test = "cargo test <vars::_cargo_all> <params>"
build = "cargo build <vars::_cargo_all>"

[scripts.check]
run = [
"# === Running all standard quality tests... ===",
"@> cargo fmt -- --check",
"@> cargo clippy --all-features --all-targets -- -D warnings",
"@> cargo test --all-features --all-targets",
"<scripts::build>",
"> <scripts::fmt>",
"> <scripts::clippy>",
"> <scripts::test>",
"# <#green>=== Completed! now you can make a PR. ===<#reset>",
]
desc = "Run a standard quality test required from make a PR."
Expand All @@ -21,8 +35,3 @@ desc = "Commands to run when exiting a session (e.g., `docker-compose down`)"
run = "# echo 'Exiting session...'"

[options.open_with]

[vars]
GREETING = "Hello from an axes variable!"

[env]
42 changes: 16 additions & 26 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,27 +1,17 @@
# Generated by Cargo
# will have compiled files and executables
debug
target
# Not included
/*
parcode-derive/target
examples/p_test.rs

# These are backup files generated by rustfmt
**/*.rs.bk

# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb

# Generated by cargo mutants
# Contains mutation testing data
**/mutants.out*/

# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
Cargo.lock
parcode.txt
road.md
.gitignore
CONTRIBUTING.md
internal_todo.md
# Included
!.axes
!benches
!examples
!parcode-derive
!src
!tests
!Cargo.toml
!LICENSE
!README.md
!TODO.md
!whitepaper.md
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "parcode"
version = "0.5.3"
version = "0.6.0"
edition = "2024"
authors = ["RetypeOS"]
description = "A high-performance, lazy load and parallelized caching library for complex Rust data structures."
Expand All @@ -26,15 +26,18 @@ exclude = [
parcode-derive = { version = "0.4.0" }
# Third
bincode = { version = "2.0.1", features = ["serde"] }
memmap2 = "0.9.9"
rayon = "1.11.0"
serde = { version = "1.0.228", features = ["derive", "rc"] }
twox-hash = "2.1.2"
# Optional
rayon = { version = "1.11.0", optional = true }
memmap2 = { version = "0.9.9", optional = true }
lz4_flex = { version = "0.12.0", optional = true }

[features]
default = []
default = ["standard"]
standard = ["parallel", "mmap"]
parallel = ["dep:rayon"]
mmap = ["dep:memmap2"]
lz4_flex = ["dep:lz4_flex"]

[dev-dependencies]
Expand Down
40 changes: 21 additions & 19 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,23 @@ Add this to your `Cargo.toml`:

```toml
[dependencies]
parcode = "0.5"
parcode = "0.6"
```

To enable LZ4 compression:

```toml
[dependencies]
parcode = { version = "0.4", features = ["lz4_flex"] }
parcode = { version = "0.6", features = ["lz4_flex"] }
```

### Features

* `mmap`: Enable memory mapping
* `parallel`: Enable parallelization
* `standard`: Default feature, activate features 'mmap' and 'parallel' (used as Default)
* `lz4_flex`: Enable LZ4 compression to ID 1.

---

## Usage Guide
Expand Down Expand Up @@ -119,10 +126,10 @@ Parcode::save("savegame.par", &world)?;
Use the builder mode.

```rust
// Saves with LZ4 compression enabled to all metadata.
// Saves with LZ4 compression on ALL and write parcode serialized data into a new file.
Parcode::builder()
.compression(true)
.write("savegame_compressed.par", &world)?;
.save("savegame_compressed.par", &world)?;
```

### 3. Read Data (Lazy)
Expand Down Expand Up @@ -178,13 +185,12 @@ Scan a list of heavy objects without loading their heavy payloads.

```rust
// Assume we have Vec<Player>
for player_proxy in world_mirror.all_players.iter()? {
let p = player_proxy?; // Resolve result
for Ok(player_proxy) in world_mirror.all_players.iter_lazy()? {

// We can check level WITHOUT loading the player's inventory from disk!
// We can check level WITHOUT loading the player's inventory of all players from disk, only you needed!
if p.level > 50 {
println!("High level player found!");
// p.inventory.load()?;
p.inventory.load()?;
}
}
```
Expand All @@ -201,21 +207,17 @@ Parcode isn't limited to files. You can serialize directly to any `std::io::Writ
let mut buffer = Vec::new();

// Serialize directly to RAM
Parcode::builder()
.compression(true)
.write_to_writer(&mut buffer, &my_data)?;
Parcode::write(&mut buffer, &my_data)?;

// 'buffer' now contains the full Parcode file structure
// 'buffer' now contains the full Parcode serialized structure
```

### Synchronous Mode

For environments where threading is not available (WASM, embedded) or to reduce memory overhead.

```rust
Parcode::builder()
.compression(true)
.write_sync("sync_save.par", &data)?;
Parcode::write_sync("sync_save.par", &data)?;
```

### Inspector
Expand Down Expand Up @@ -246,12 +248,12 @@ Root Offset: 550368

Control exactly how your data structure maps to disk using `#[parcode(...)]`.

| Attribute | Effect | Best For |
| :------------------------------ | :--------------------------------------------- | :--------------------------------- |
| **(none)** | Field is serialized into the parent's payload. | Small primitives (`u32`, `bool`), short Strings, flags. |
| Attribute | Effect | Best For |
| :------------------------------ | :--------------------------------------------- | :------------------------------------------------------ |
| **(none)** | Field is serialized into the parent's payload. | Small primitives (`u32`, `bool`), short Strings, flags. |
| `#[parcode(chunkable)]` | Field is stored in its own independent Chunk. | Structs, Vectors, or fields you want to load lazily (`.load()`). |
| `#[parcode(map)]` | Field (`HashMap`) is sharded by hash. | Large Dictionaries/Indices where you need random access (`.get()`). |
| `#[parcode(compression="lz4")]` | Overrides compression for this chunk. | Highly compressible data (text, save states). |
| `#[parcode(compression="lz4")]` | Overrides compression for this chunk. | Highly compressible data (text, save states). |

---

Expand Down
6 changes: 3 additions & 3 deletions examples/memory_buffer.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Example: Writing Parcode to a Memory Buffer
//!
//! This example demonstrates how to use the generic `write_to_writer` API
//! This example demonstrates how to use the generic `write` API
//! to serialize a Parcode object directly into a `Vec<u8>` in memory,
//! bypassing the file system.

Expand Down Expand Up @@ -36,12 +36,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut buffer: Vec<u8> = Vec::new();

// 3. Serialize to the buffer
// We use `write_to_writer` which accepts any W: Write + Send
// We use `write` which accepts any W: Write + Send
// Vec<u8> implements Write and Send.
println!("Serializing to memory buffer...");
Parcode::builder()
.compression(true) // Optional: enable compression
.write_to_writer(&mut buffer, &user)?;
.write(&mut buffer, &user)?;

println!(
"Serialization complete. Buffer size: {} bytes",
Expand Down
Loading