diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5ab1a4..15730f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,5 +44,8 @@ jobs: - name: Build release run: cargo build --workspace --release + - name: Run benchmark smoke + run: cargo bench -p manas-benches -- --quick + - name: Smoke test CLI binary run: ./target/release/manas --help diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 655e1d6..6b9b9d1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,7 +35,8 @@ 14. [The Importance Scoring System](#14-the-importance-scoring-system) 15. [The Freshness System](#15-the-freshness-system) 16. [Error Handling Strategy](#16-error-handling-strategy) -17. [What Manas Is Not](#17-what-manas-is-not) +17. [Benchmarks and Integration Gates](#17-benchmarks-and-integration-gates) +18. [What Manas Is Not](#18-what-manas-is-not) --- @@ -450,13 +451,19 @@ manas/ │ ├── toml.rs │ └── csv.rs │ -└── manas-cli/ ← USER INTERFACE +├── manas-cli/ ← USER INTERFACE ├── Cargo.toml ← deps: all crates above └── src/ └── main.rs ← std-only arg parsing, command routing, formatting +│ +└── manas-benches/ ← TOOLING ONLY + └── benches/ + └── bench.rs ← B1-B8 benchmark harness, BENCHMARKS.md generator ``` -**Total crates: 5** (v1 had 9 — simpler is better) +**Runtime crates: 5** (v1 had 9 — simpler is better) + +**Tooling crates: 1** (`manas-benches`, not part of the runtime path) **No `manas-language` crate** — the transformer path from v1 is removed. Language generation is a future milestone, not the foundation. @@ -1178,7 +1185,44 @@ impl std::error::Error for ManasError { ... } --- -## 17. What Manas Is Not +## 17. Benchmarks and Integration Gates + +Stage 16 keeps performance and regression claims measurable. + +`manas-benches` is a dedicated non-runtime crate with a custom no-dependency +benchmark harness: + +| ID | What it measures | +|---|---| +| B1 | Single `teach` call | +| B2 | Single `ask` call | +| B3 | `.manas` save | +| B4 | `.manas` load | +| B5 | Tokenizer throughput on 1000 words | +| B6 | Full anti-forgetting proof | +| B7 | Estimated 1000-neuron memory footprint | +| B8 | Brain file growth per new fact | + +The committed benchmark report is generated with: + +```bash +cargo bench -p manas-benches -- --write-markdown BENCHMARKS.md +``` + +CI runs the quick benchmark smoke: + +```bash +cargo bench -p manas-benches -- --quick +``` + +The Stage 16 integration gate lives in `manas-cli/tests/stage16_integration.rs` +because the workspace root is virtual and `cargo test --workspace` only runs +tests attached to workspace packages. It covers the real demo, anti-forgetting, +persistence, growth, protection, compression, freshness, and ingestion. + +--- + +## 18. What Manas Is Not Manas v2 is honest about its scope: diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..b8224e8 --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,21 @@ +# Benchmarks + +Generated by `cargo bench -p manas-benches -- --write-markdown BENCHMARKS.md`. + +Mode: `full` + +| ID | Benchmark | Result | Unit | Detail | +|---|---|---:|---|---| +| B1 | single teach | 0.0384 | ms/op | 25 iterations | +| B2 | single ask | 0.0190 | ms/op | 200 iterations | +| B3 | .manas save | 0.4677 | ms/op | 25 iterations | +| B4 | .manas load | 0.1514 | ms/op | 50 iterations | +| B5 | tokenizer 1000 words | 0.3946 | ms/op | 100 iterations | +| B6 | anti-forgetting proof | 0.3630 | s | single fixed seed | +| B7 | 1000-neuron footprint | 419.6875 | KiB | estimated heap footprint, total=1000 | +| B8 | brain growth per fact | 768.8750 | bytes/fact | n=32, min=646, max=4534 | + +B7 reports an internal heap-footprint estimate for network-owned buffers and neuron storage, not process RSS. + +Run full benchmarks with `cargo bench -p manas-benches`. +Run CI smoke benchmarks with `cargo bench -p manas-benches -- --quick`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2659b62..2faa1a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Manas uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Stage 13 — The real demo. - Stage 14 — Inspect, neurons, and debug commands. - Stage 15 — Compression and forget command. +- Stage 16 — Benchmarks and test suite. ### Added @@ -105,10 +106,18 @@ Manas uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Added compression tests proving neuron count reduction, frozen-neuron safety, high-importance survival, anti-forgetting after compression, dry-run behavior, and brain file-size shrinkage. +- Added the non-runtime `manas-benches` workspace crate with B1-B8 benchmark + coverage for teach, ask, save, load, tokenization, anti-forgetting, memory + footprint, and brain file growth. +- Added `BENCHMARKS.md` with generated full-run benchmark results. +- Added Stage 16 integration coverage for the 22-fact demo, anti-forgetting, + persistence, growth, protection, compression, freshness, and ingestion. +- Added CI benchmark smoke coverage through + `cargo bench -p manas-benches -- --quick`. ### Next -- Stage 16 — Benchmarks and test suite. +- Stage 17 — Layer growth. --- diff --git a/Cargo.lock b/Cargo.lock index fa4f2bf..363514d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "manas-benches" +version = "0.1.0" +dependencies = [ + "manas-core", + "manas-learn", + "manas-store", +] + [[package]] name = "manas-cli" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a3e311f..477c58b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "manas-learn", "manas-ingest", "manas-cli", + "manas-benches", ] [workspace.package] diff --git a/README.md b/README.md index 8281d2b..af0f8b3 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,8 @@ Manas v2 is in active development. The roadmap follows a strict rule: | Stage 13 | The real demo | Complete | | Stage 14 | Inspect, neurons, and debug commands | Complete | | Stage 15 | Compression and forget command | Complete | -| Stage 16+ | Benchmarks, layer growth, future agents | Planned | +| Stage 16 | Benchmarks and test suite | Complete | +| Stage 17+ | Layer growth, future agents | Planned | Stages 1 and 2 are preserved as a standalone proof in `manas-core/src/experiment.rs`. Stage 3 promotes the proven engine into @@ -163,6 +164,10 @@ inspectable with rich `inspect`, `neurons`, and `trace` commands for debugging learned weights, metadata, and query flow. Stage 15 adds conservative compression through `manas forget`, merging only stale low-importance `Open` hidden neurons that have a highly similar retained neighbor. +Stage 16 adds a dedicated benchmark crate, committed benchmark report, CI +benchmark smoke test, and an eight-test integration gate covering the demo, +anti-forgetting, persistence, growth, protection, compression, freshness, and +ingestion. Run the proof: @@ -236,6 +241,19 @@ cargo test -p manas-core compression cargo test -p manas-learn compression ``` +Run the Stage 16 integration proof: + +```bash +cargo test -p manas-cli stage16 +``` + +Run the benchmarks: + +```bash +cargo bench -p manas-benches -- --quick +cargo bench -p manas-benches -- --write-markdown BENCHMARKS.md +``` + Run the CLI proof: ```bash @@ -249,6 +267,7 @@ Run the ingestion proof: cargo test -p manas-ingest ``` +See [BENCHMARKS.md](./BENCHMARKS.md) for the latest committed benchmark run. See [ROADMAP.md](./ROADMAP.md) for the full plan with tests. See [ARCHITECTURE.md](./ARCHITECTURE.md) for the full design. @@ -281,7 +300,7 @@ cargo build --workspace --release ## Architecture -Manas v2 is built from 5 Rust crates, each with a single responsibility: +Manas v2 is built from 5 runtime Rust crates plus one benchmark tooling crate: ``` ┌──────────────────────────────────────────┐ @@ -325,6 +344,9 @@ Manas v2 is built from 5 Rust crates, each with a single responsibility: The transformer path from v1 is removed. The text sidecar from v1 is removed. The answering system from v1 is replaced with direct neural weight retrieval. +`manas-benches` is a non-runtime workspace crate used for Stage 16 benchmark +measurement and CI smoke coverage. + --- ## The `.manas` File Format diff --git a/ROADMAP.md b/ROADMAP.md index 4c4b79f..f4a4615 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -84,7 +84,7 @@ from Stage 2 onward.** | Stage 13 | The real demo | Complete | | Stage 14 | Inspect, neurons, and debug commands | Complete | | Stage 15 | Compression and forget command | Complete | -| Stage 16 | Benchmarks and test suite | Planned | +| Stage 16 | Benchmarks and test suite | Complete | | Stage 17 | Layer growth | Planned | | Stage 18 | Internet agent (future) | Planned | | Stage 19 | Language generation (future) | Planned | @@ -1949,7 +1949,7 @@ fn anti_forgetting_test_still_passes_after_compression() { ### Integration Tests ```rust -// tests/integration/ +// manas-cli/tests/stage16_integration.rs // IT-1: 22-fact demo test (Stage 13) runs in CI // IT-2: Anti-forgetting: 5 facts survive 50 new facts @@ -1973,15 +1973,29 @@ jobs: - cargo test --workspace - cargo clippy --workspace -- -D warnings - cargo fmt --check + - cargo bench -p manas-benches -- --quick ``` ### Done When -- [ ] All 8 benchmarks produce stable numbers -- [ ] All 8 integration tests pass in CI -- [ ] `cargo test --workspace` passes with zero failures -- [ ] `cargo clippy` passes with zero warnings -- [ ] Benchmark results documented in `BENCHMARKS.md` +- [x] All 8 benchmarks produce stable numbers +- [x] All 8 integration tests pass in CI +- [x] `cargo test --workspace` passes with zero failures +- [x] `cargo clippy` passes with zero warnings +- [x] Benchmark results documented in `BENCHMARKS.md` + +### Stage 16 Implementation Notes + +- Added the non-runtime `manas-benches` workspace crate with a deterministic + custom benchmark harness covering B1 through B8. +- Added `BENCHMARKS.md`, generated by + `cargo bench -p manas-benches -- --write-markdown BENCHMARKS.md`. +- Added the `manas-cli/tests/stage16_integration.rs` integration gate covering + IT-1 through IT-8. +- Added CI benchmark smoke coverage with + `cargo bench -p manas-benches -- --quick`. +- Kept Stage 16 coverage inside the existing crates so the runtime architecture + remains unchanged. --- diff --git a/manas-benches/Cargo.toml b/manas-benches/Cargo.toml new file mode 100644 index 0000000..8b56357 --- /dev/null +++ b/manas-benches/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "manas-benches" +version.workspace = true +edition.workspace = true +publish = false + +[[bench]] +name = "bench" +harness = false + +[dependencies] +manas-core = { workspace = true } +manas-store = { workspace = true } +manas-learn = { workspace = true } diff --git a/manas-benches/benches/bench.rs b/manas-benches/benches/bench.rs new file mode 100644 index 0000000..1cbc363 --- /dev/null +++ b/manas-benches/benches/bench.rs @@ -0,0 +1,438 @@ +use std::env; +use std::fs; +use std::hint::black_box; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use manas_core::{Network, ProtectionLevel}; +use manas_learn::fixtures::{ + ANCHOR_FACTS, ANCHOR_NEURONS_PER_FACT, ANCHOR_TRAIN_EPOCHS, EMBED_DIM, HIDDEN_DIM, + LEARNING_RATE, NOISE_FACTS, NOISE_TRAIN_EPOCHS, OUTPUT_DIM, +}; +use manas_learn::{Tokenizer, Trainer}; +use manas_store::{BrainState, ManasBrain}; + +const KB: f64 = 1024.0; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Mode { + Quick, + Full, +} + +#[derive(Clone, Debug)] +struct BenchOptions { + mode: Mode, + write_markdown: Option, +} + +#[derive(Clone, Debug)] +struct BenchResult { + id: &'static str, + name: &'static str, + value: f64, + unit: &'static str, + detail: String, +} + +fn main() { + let options = parse_options(); + let results = run_benchmarks(options.mode); + let markdown = render_markdown(options.mode, &results); + println!("{markdown}"); + + if let Some(path) = options.write_markdown { + let report_path = resolve_report_path(&path); + fs::write(&report_path, markdown).unwrap_or_else(|error| { + panic!("failed to write {}: {error}", report_path.display()); + }); + } +} + +fn run_benchmarks(mode: Mode) -> Vec { + vec![ + bench_single_teach(mode), + bench_single_ask(mode), + bench_save(mode), + bench_load(mode), + bench_tokenizer_1000_words(mode), + bench_anti_forgetting(), + bench_memory_1000_neurons(), + bench_file_growth_per_fact(mode), + ] +} + +fn bench_single_teach(mode: Mode) -> BenchResult { + let iterations = iterations(mode, 5, 25); + let elapsed = repeat(iterations, || { + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + trainer + .learn( + &mut network, + "cat", + "small domesticated animal with fur and whiskers", + ) + .expect("teach benchmark should learn"); + black_box(network.neuron_count()); + }); + + BenchResult { + id: "B1", + name: "single teach", + value: millis_per_iter(elapsed, iterations), + unit: "ms/op", + detail: format!("{iterations} iterations"), + } +} + +fn bench_single_ask(mode: Mode) -> BenchResult { + let iterations = iterations(mode, 25, 200); + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + trainer + .learn( + &mut network, + "cat", + "small domesticated animal with fur and whiskers", + ) + .expect("ask benchmark setup should learn"); + + let elapsed = repeat(iterations, || { + let result = trainer + .query(&network, "What is a cat?") + .expect("ask benchmark should query"); + black_box(result.confidence); + }); + + BenchResult { + id: "B2", + name: "single ask", + value: millis_per_iter(elapsed, iterations), + unit: "ms/op", + detail: format!("{iterations} iterations"), + } +} + +fn bench_save(mode: Mode) -> BenchResult { + let iterations = iterations(mode, 5, 25); + let (state, _trainer) = trained_state(12); + let dir = temp_dir("bench-save"); + let brain = ManasBrain::new(dir.join("brain.manas")); + + let elapsed = repeat(iterations, || { + brain + .save_state(&state) + .expect("save benchmark should write state"); + black_box(brain.size_bytes()); + }); + + cleanup_dir(&dir); + BenchResult { + id: "B3", + name: ".manas save", + value: millis_per_iter(elapsed, iterations), + unit: "ms/op", + detail: format!("{iterations} iterations"), + } +} + +fn bench_load(mode: Mode) -> BenchResult { + let iterations = iterations(mode, 10, 50); + let (state, _trainer) = trained_state(12); + let dir = temp_dir("bench-load"); + let brain = ManasBrain::new(dir.join("brain.manas")); + brain + .save_state(&state) + .expect("load benchmark setup should save state"); + + let elapsed = repeat(iterations, || { + let loaded = brain + .load_state() + .expect("load benchmark should load state"); + black_box(loaded.network.neuron_count()); + }); + + cleanup_dir(&dir); + BenchResult { + id: "B4", + name: ".manas load", + value: millis_per_iter(elapsed, iterations), + unit: "ms/op", + detail: format!("{iterations} iterations"), + } +} + +fn bench_tokenizer_1000_words(mode: Mode) -> BenchResult { + let iterations = iterations(mode, 10, 100); + let text = thousand_word_text(); + let elapsed = repeat(iterations, || { + let mut tokenizer = Tokenizer::new(4); + let ids = tokenizer.encode(&text); + black_box(ids.len()); + }); + + BenchResult { + id: "B5", + name: "tokenizer 1000 words", + value: millis_per_iter(elapsed, iterations), + unit: "ms/op", + detail: format!("{iterations} iterations"), + } +} + +fn bench_anti_forgetting() -> BenchResult { + let elapsed = measure(|| { + let mut trainer = Trainer::with_seed(42 ^ 0x6a09_e667_f3bc_c909, EMBED_DIM, LEARNING_RATE); + let anchors = trainer.encode_facts(&ANCHOR_FACTS); + let noise = trainer.encode_facts(&NOISE_FACTS); + let mut network = Network::with_seed( + 42 ^ 0xbb67_ae85_84ca_a73b, + EMBED_DIM, + HIDDEN_DIM, + OUTPUT_DIM, + ); + + trainer + .train_facts(&mut network, &anchors, ANCHOR_TRAIN_EPOCHS) + .expect("anchor training should succeed"); + trainer + .consolidate_anchors(&mut network, &anchors, ANCHOR_NEURONS_PER_FACT) + .expect("anchor consolidation should succeed"); + trainer + .train_facts(&mut network, &noise, NOISE_TRAIN_EPOCHS) + .expect("noise training should succeed"); + trainer + .fit_new_facts(&mut network, &noise, &anchors) + .expect("new fact fitting should succeed"); + black_box(network.neuron_count()); + }); + + BenchResult { + id: "B6", + name: "anti-forgetting proof", + value: elapsed.as_secs_f64(), + unit: "s", + detail: "single fixed seed".to_string(), + } +} + +fn bench_memory_1000_neurons() -> BenchResult { + let network = Network::new(32, 968, 32); + let bytes = estimate_network_bytes(&network); + + BenchResult { + id: "B7", + name: "1000-neuron footprint", + value: bytes as f64 / KB, + unit: "KiB", + detail: format!("estimated heap footprint, total={}", network.neuron_count()), + } +} + +fn bench_file_growth_per_fact(mode: Mode) -> BenchResult { + let fact_count = match mode { + Mode::Quick => 8, + Mode::Full => 32, + }; + let dir = temp_dir("bench-growth"); + let brain = ManasBrain::new(dir.join("brain.manas")); + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + let mut previous_size = 0_u64; + let mut deltas = Vec::with_capacity(fact_count); + + for index in 0..fact_count { + let input = format!("bench fact {index}"); + let target = format!("bench value {index}"); + trainer + .learn(&mut network, &input, &target) + .expect("file growth benchmark should learn"); + let state = BrainState::new(network.clone(), store_vocab(&trainer)); + brain + .save_state(&state) + .expect("file growth benchmark should save"); + let size = brain.size_bytes(); + deltas.push(size.saturating_sub(previous_size)); + previous_size = size; + } + + cleanup_dir(&dir); + let average = deltas.iter().sum::() as f64 / deltas.len() as f64; + let min = deltas.iter().min().copied().unwrap_or(0); + let max = deltas.iter().max().copied().unwrap_or(0); + + BenchResult { + id: "B8", + name: "brain growth per fact", + value: average, + unit: "bytes/fact", + detail: format!("n={fact_count}, min={min}, max={max}"), + } +} + +fn trained_state(fact_count: usize) -> (BrainState, Trainer) { + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + + for index in 0..fact_count { + let input = format!("benchmark fact {index}"); + let target = format!("benchmark value {index}"); + trainer + .learn(&mut network, &input, &target) + .expect("benchmark setup should learn"); + } + + (BrainState::new(network, store_vocab(&trainer)), trainer) +} + +fn store_vocab(trainer: &Trainer) -> Vec { + trainer + .encoder + .export_vocab() + .into_iter() + .map(|entry| manas_store::VocabEntry { + token: entry.token, + id: entry.id, + embedding: entry.embedding, + }) + .collect() +} + +fn estimate_network_bytes(network: &Network) -> usize { + network + .layers + .iter() + .map(|layer| { + layer.neurons.capacity() * std::mem::size_of::() + + layer + .neurons + .iter() + .map(estimate_neuron_owned_bytes) + .sum::() + }) + .sum() +} + +fn estimate_neuron_owned_bytes(neuron: &manas_core::Neuron) -> usize { + neuron.weights.capacity() * std::mem::size_of::() + + neuron.weight_protection.capacity() * std::mem::size_of::() +} + +fn thousand_word_text() -> String { + let words = [ + "cat", "paris", "rust", "everest", "dna", "amazon", "bitcoin", "jupiter", "compiler", + "network", + ]; + (0..1000) + .map(|index| words[index % words.len()]) + .collect::>() + .join(" ") +} + +fn repeat(iterations: usize, mut body: impl FnMut()) -> Duration { + let start = Instant::now(); + for _ in 0..iterations { + body(); + } + start.elapsed() +} + +fn measure(body: impl FnOnce()) -> Duration { + let start = Instant::now(); + body(); + start.elapsed() +} + +fn millis_per_iter(elapsed: Duration, iterations: usize) -> f64 { + elapsed.as_secs_f64() * 1000.0 / iterations as f64 +} + +fn iterations(mode: Mode, quick: usize, full: usize) -> usize { + match mode { + Mode::Quick => quick, + Mode::Full => full, + } +} + +fn render_markdown(mode: Mode, results: &[BenchResult]) -> String { + let mode_label = match mode { + Mode::Quick => "quick", + Mode::Full => "full", + }; + let mut output = String::new(); + output.push_str("# Benchmarks\n\n"); + output.push_str( + "Generated by `cargo bench -p manas-benches -- --write-markdown BENCHMARKS.md`.\n\n", + ); + output.push_str(&format!("Mode: `{mode_label}`\n\n")); + output.push_str("| ID | Benchmark | Result | Unit | Detail |\n"); + output.push_str("|---|---|---:|---|---|\n"); + for result in results { + output.push_str(&format!( + "| {} | {} | {:.4} | {} | {} |\n", + result.id, result.name, result.value, result.unit, result.detail + )); + } + output.push('\n'); + output.push_str("B7 reports an internal heap-footprint estimate for network-owned buffers and neuron storage, not process RSS.\n\n"); + output.push_str("Run full benchmarks with `cargo bench -p manas-benches`.\n"); + output.push_str("Run CI smoke benchmarks with `cargo bench -p manas-benches -- --quick`.\n"); + output +} + +fn parse_options() -> BenchOptions { + let mut mode = Mode::Full; + let mut write_markdown = None; + let mut args = env::args().skip(1); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--bench" => {} + "--quick" => mode = Mode::Quick, + "--write-markdown" => { + let path = args + .next() + .unwrap_or_else(|| panic!("--write-markdown requires a path")); + write_markdown = Some(PathBuf::from(path)); + } + "--help" | "-h" => { + println!( + "Usage: cargo bench -p manas-benches -- [--quick] [--write-markdown PATH]" + ); + std::process::exit(0); + } + other => panic!("unknown benchmark option '{other}'"), + } + } + + BenchOptions { + mode, + write_markdown, + } +} + +fn resolve_report_path(path: &Path) -> PathBuf { + if path.is_absolute() { + return path.to_path_buf(); + } + + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("bench crate should live under workspace root") + .join(path) +} + +fn temp_dir(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after unix epoch") + .as_nanos(); + let dir = env::temp_dir().join(format!("manas-{name}-{}-{nanos}", std::process::id())); + fs::create_dir_all(&dir).expect("benchmark temp dir should be created"); + dir +} + +fn cleanup_dir(path: &Path) { + let _ = fs::remove_dir_all(path); +} diff --git a/manas-cli/tests/stage16_integration.rs b/manas-cli/tests/stage16_integration.rs new file mode 100644 index 0000000..b36e009 --- /dev/null +++ b/manas-cli/tests/stage16_integration.rs @@ -0,0 +1,401 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use manas_core::{Network, NeuronGradients, ProtectionLevel, Source}; +use manas_ingest::{IngestSource, ingest}; +use manas_learn::fixtures::{ + ANCHOR_FACTS, ANCHOR_NEURONS_PER_FACT, ANCHOR_SURVIVAL_THRESHOLD, ANCHOR_TRAIN_EPOCHS, + EMBED_DIM, HIDDEN_DIM, LEARNING_RATE, MAX_FORGETTING_DELTA, NEW_FACT_THRESHOLD, NOISE_FACTS, + NOISE_TRAIN_EPOCHS, OUTPUT_DIM, +}; +use manas_learn::{EncodedFact, FreshnessCategory, Trainer}; +use manas_store::{BrainState, ManasBrain, VocabEntry}; + +const DEMO_FACTS: &[&str] = &[ + "A cat is a small domesticated animal with fur and whiskers.", + "The Eiffel Tower is located in Paris France and was built in 1889.", + "The Amazon River is the largest river by discharge in the world.", + "Photosynthesis is the process by which plants convert sunlight into energy.", + "Hydrogen is the lightest and most abundant element in the universe.", + "The human brain contains approximately 86 billion neurons.", + "Mount Everest is the highest mountain on Earth at 8849 meters.", + "Shakespeare wrote 37 plays and 154 sonnets during his lifetime.", + "The speed of light in vacuum is approximately 299792458 meters per second.", + "DNA is a double helix structure that carries genetic information.", + "The Roman Empire fell in 476 AD when Romulus Augustulus was deposed.", + "Water boils at 100 degrees Celsius at standard atmospheric pressure.", + "The Python programming language was created by Guido van Rossum in 1991.", + "Jupiter is the largest planet in our solar system with 95 known moons.", + "The Mona Lisa was painted by Leonardo da Vinci in the early 16th century.", + "Rust programming language was first released by Mozilla Research in 2010.", + "The mitochondria is the powerhouse of the cell in biology.", + "Albert Einstein developed the theory of relativity in the early 20th century.", + "The Pacific Ocean is the largest and deepest ocean on Earth.", + "Bitcoin was created by Satoshi Nakamoto and launched in January 2009.", + "The nitrogen cycle describes how nitrogen moves through ecosystems.", + "Gravity pulls objects toward each other with a force proportional to mass.", +]; + +#[test] +fn stage16_it_1_demo_answers_from_neural_weights_only() { + let dir = temp_dir("it1-demo"); + + assert_success(&run(&dir, &["reset"])); + for fact in DEMO_FACTS { + assert_success(&run(&dir, &["teach", fact])); + } + remove_sidecars(&dir); + + for (question, expected_words) in [ + ( + "What is a cat?", + &["small", "domesticated", "animal", "fur", "whiskers"][..], + ), + ( + "Where is the Eiffel Tower?", + &["located", "paris", "france", "built", "1889"][..], + ), + ( + "What did Einstein develop?", + &["theory", "relativity", "early", "20th", "century"][..], + ), + ] { + let ask = run(&dir, &["ask", question]); + assert_success(&ask); + let output = stdout(&ask); + assert!( + output.contains("Answered from\n neural weights"), + "{output}" + ); + for word in expected_words { + assert!(output.to_lowercase().contains(word), "{output}"); + } + } + + cleanup_dir(dir); +} + +#[test] +fn stage16_it_2_anti_forgetting_anchor_survival() { + let mut trainer = Trainer::with_seed(42 ^ 0x6a09_e667_f3bc_c909, EMBED_DIM, LEARNING_RATE); + let anchors = trainer.encode_facts(&ANCHOR_FACTS); + let noise = trainer.encode_facts(&NOISE_FACTS); + let mut network = Network::with_seed( + 42 ^ 0xbb67_ae85_84ca_a73b, + EMBED_DIM, + HIDDEN_DIM, + OUTPUT_DIM, + ); + + trainer + .train_facts(&mut network, &anchors, ANCHOR_TRAIN_EPOCHS) + .unwrap(); + trainer + .consolidate_anchors(&mut network, &anchors, ANCHOR_NEURONS_PER_FACT) + .unwrap(); + let before = score_facts(&trainer, &network, &anchors); + trainer + .train_facts(&mut network, &noise, NOISE_TRAIN_EPOCHS) + .unwrap(); + trainer + .fit_new_facts(&mut network, &noise, &anchors) + .unwrap(); + + let after = score_facts(&trainer, &network, &anchors); + let new_scores = score_facts(&trainer, &network, &noise); + for (index, (before_score, after_score)) in before.iter().zip(after.iter()).enumerate() { + assert!( + *after_score >= ANCHOR_SURVIVAL_THRESHOLD, + "anchor {index} below threshold: before {before_score:.4}, after {after_score:.4}" + ); + assert!( + before_score - after_score <= MAX_FORGETTING_DELTA, + "anchor {index} forgot too much: before {before_score:.4}, after {after_score:.4}" + ); + } + assert!(new_scores.iter().all(|score| *score >= NEW_FACT_THRESHOLD)); +} + +#[test] +fn stage16_it_3_persistence_survives_save_load_save_load_cycle() { + let dir = temp_dir("it3-persistence"); + let path = dir.join("brain.manas"); + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + + trainer + .learn(&mut network, "cat", "small domesticated animal with fur") + .unwrap(); + let state = BrainState::new(network, store_vocab(&trainer)); + let brain = ManasBrain::new(path); + brain.save_state(&state).unwrap(); + + let first = brain.load_state().unwrap(); + brain.save_state(&first).unwrap(); + let second = brain.load_state().unwrap(); + + assert_eq!(second.vocab_entries.len(), first.vocab_entries.len()); + assert_eq!(second.network.neuron_count(), first.network.neuron_count()); + assert_eq!(second.network.input_dim, first.network.input_dim); + cleanup_dir(dir); +} + +#[test] +fn stage16_it_4_growth_handles_novel_and_repeated_input() { + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + + trainer.learn(&mut network, "cat", "animal").unwrap(); + assert!(network.neuron_count() > 0); + let after_first = network.neuron_count(); + + for _ in 0..10 { + trainer.learn(&mut network, "cat", "animal").unwrap(); + } + let after_repeated = network.neuron_count(); + assert_eq!(after_first, after_repeated); + + trainer + .learn(&mut network, "eiffel tower", "located paris france") + .unwrap(); + assert!(network.neuron_count() >= after_repeated); +} + +#[test] +fn stage16_it_5_frozen_components_survive_stress_updates() { + let mut network = Network::new(EMBED_DIM, HIDDEN_DIM, OUTPUT_DIM); + network.layers[0].neurons[0].freeze_all(); + network.layers[1].neurons[0].weight_protection[0] = ProtectionLevel::Frozen; + let frozen_hidden = network.layers[0].neurons[0].weights.clone(); + let frozen_edge = network.layers[1].neurons[0].weights[0]; + + let gradients = vec![ + ( + network.layers[0].neurons[0].id, + NeuronGradients { + weight_gradients: vec![10.0; EMBED_DIM], + bias_gradient: 10.0, + }, + ), + ( + network.layers[1].neurons[0].id, + NeuronGradients { + weight_gradients: vec![10.0; HIDDEN_DIM], + bias_gradient: 10.0, + }, + ), + ]; + + for _ in 0..1000 { + network.apply_gradients(&gradients, 1.0).unwrap(); + } + + assert_eq!(network.layers[0].neurons[0].weights, frozen_hidden); + assert_eq!(network.layers[1].neurons[0].weights[0], frozen_edge); +} + +#[test] +fn stage16_it_6_forget_shrinks_brain_file() { + let dir = temp_dir("it6-forget"); + write_compressible_brain(&dir); + let path = dir.join("brain.manas"); + let size_before = fs::metadata(&path).unwrap().len(); + + let forget = run(&dir, &["forget", "--threshold", "0.20"]); + assert_success(&forget); + let output = stdout(&forget); + assert!(output.contains("neurons removed : 1"), "{output}"); + + let size_after = fs::metadata(&path).unwrap().len(); + assert!(size_after < size_before, "{size_before} -> {size_after}"); + cleanup_dir(dir); +} + +#[test] +fn stage16_it_7_stale_fast_fact_warns_on_ask() { + let dir = temp_dir("it7-freshness"); + let brain = ManasBrain::new(dir.join("brain.manas")); + let mut network = Network::new_empty(EMBED_DIM); + let mut trainer = Trainer::with_seed(42, EMBED_DIM, LEARNING_RATE); + + trainer + .learn_with_source_and_freshness( + &mut network, + "rust version", + "released last month", + Source::RawText, + FreshnessCategory::Fast, + ) + .unwrap(); + let stale_born_at = unix_now().saturating_sub(31 * 86_400); + network.layers[0].neurons[0].born_at = stale_born_at; + network.layers[0].neurons[0].last_activated = stale_born_at; + brain + .save_state(&BrainState::new(network, store_vocab(&trainer))) + .unwrap(); + + let ask = run(&dir, &["ask", "What is rust version?"]); + assert_success(&ask); + let output = stdout(&ask); + assert!( + output.contains("This knowledge may be outdated"), + "{output}" + ); + cleanup_dir(dir); +} + +#[test] +fn stage16_it_8_ingestion_teaches_txt_md_and_rs_sources() { + let dir = temp_dir("it8-ingest"); + let docs = dir.join("docs"); + fs::create_dir_all(&docs).unwrap(); + fs::write(docs.join("cat.txt"), "A cat is a small animal with fur.").unwrap(); + fs::write( + docs.join("paris.md"), + "# Paris\n\nParis is a city in France.", + ) + .unwrap(); + fs::write( + docs.join("code.rs"), + "/// Rust is a systems language.\nfn main() {}", + ) + .unwrap(); + + let chunks = ingest(IngestSource::Folder(docs.clone())).unwrap(); + assert!(chunks.iter().any(|chunk| matches!( + &chunk.source, + Source::LocalFile { path } if path.ends_with("cat.txt") + ))); + assert!(chunks.iter().any(|chunk| matches!( + &chunk.source, + Source::LocalFile { path } if path.ends_with("paris.md") + ))); + assert!(chunks.iter().any(|chunk| matches!( + &chunk.source, + Source::LocalFile { path } if path.ends_with("code.rs") + ))); + + assert_success(&run(&dir, &["teach", "docs"])); + let ask = run(&dir, &["ask", "Where is Paris?"]); + assert_success(&ask); + assert!(stdout(&ask).contains("neural weights")); + cleanup_dir(dir); +} + +fn score_facts(trainer: &Trainer, network: &Network, facts: &[EncodedFact]) -> Vec { + facts + .iter() + .map(|fact| trainer.similarity_for_fact(network, fact)) + .collect() +} + +fn store_vocab(trainer: &Trainer) -> Vec { + trainer + .encoder + .export_vocab() + .into_iter() + .map(|entry| VocabEntry { + token: entry.token, + id: entry.id, + embedding: entry.embedding, + }) + .collect() +} + +fn write_compressible_brain(dir: &Path) { + let mut network = Network::new_empty(4); + for _ in 0..3 { + network.grow_neuron(0, 4).unwrap(); + } + + let now = unix_now(); + let day = 86_400; + network.layers[0].neurons[0].weights = vec![1.0, 0.0, 0.0, 0.0]; + network.layers[0].neurons[0].guard_all(); + network.layers[0].neurons[0].importance_score = 0.75; + network.layers[0].neurons[0].last_activated = now; + network.layers[0].neurons[0].born_at = now - 2 * day; + + network.layers[0].neurons[1].weights = vec![1.0, 0.0, 0.0, 0.0]; + network.layers[0].neurons[1].importance_score = 0.01; + network.layers[0].neurons[1].last_activated = now - 31 * day; + network.layers[0].neurons[1].born_at = now - 60 * day; + + network.layers[0].neurons[2].weights = vec![0.0, 1.0, 0.0, 0.0]; + network.layers[0].neurons[2].importance_score = 0.80; + network.layers[0].neurons[2].last_activated = now; + network.layers[0].neurons[2].born_at = now - 2 * day; + + for output_neuron in &mut network.layers[1].neurons { + output_neuron.weights = vec![0.25, 0.05, 0.10]; + output_neuron.weight_protection = vec![ProtectionLevel::Open; 3]; + } + + ManasBrain::new(dir.join("brain.manas")) + .save_state(&BrainState::new(network, Vec::new())) + .unwrap(); +} + +fn remove_sidecars(dir: &Path) { + for sidecar in [ + "brain.manas.sources", + "brain.manas.sourceindex", + "brain.manas.seq", + "brain.manas.transformer", + "brain.manas.langmeta", + ] { + let _ = fs::remove_file(dir.join(sidecar)); + } +} + +fn run(dir: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_manas")) + .args(args) + .current_dir(dir) + .output() + .unwrap() +} + +fn assert_success(output: &Output) { + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + stdout(output), + stderr(output) + ); +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +fn temp_dir(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "manas-stage16-{name}-{}-{nanos}", + std::process::id() + )); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn cleanup_dir(path: PathBuf) { + let _ = fs::remove_dir_all(path); +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +}