From d61c28ac4155d5f9851dd09968bf1cb984feb4da Mon Sep 17 00:00:00 2001 From: Felipe705x Date: Mon, 6 Jul 2026 23:23:34 -0400 Subject: [PATCH 1/2] Add pattern_compare: typecheck timing on reduced path patterns Times `check_query` on the internal-bench cases reduced to the FPPC path-pattern surface (MATCH/RETURN dropped, query-level WHERE inlined), isolated from the runtime with a reused checker and 1k warmup / 10k iters. Optionally joins the numbers against a reference CSV to print a per-case ratio and geomean, and sanity-checks that each verdict (valid/empty/invalid) matches the reference. Self-contained: default dataset path bench/data/ldbc-sf0.1.gdb, reference CSV passed as an argument (no comparison printed when absent). Only pulls always-on library deps, so it builds under any feature set. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/bin/pattern_compare.rs | 203 +++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 src/bin/pattern_compare.rs diff --git a/src/bin/pattern_compare.rs b/src/bin/pattern_compare.rs new file mode 100644 index 0000000..9411693 --- /dev/null +++ b/src/bin/pattern_compare.rs @@ -0,0 +1,203 @@ +//! pattern_compare — times the type checker on the reduced LDBC path-pattern +//! cases and, optionally, compares against a reference CSV. +//! +//! Each case is an internal-bench query reduced to the FPPC path-pattern surface +//! (MATCH / RETURN dropped, a query-level WHERE inlined into the descriptor). +//! Rust times `check_query` on the reduced pattern — i.e. the path-pattern +//! typecheck only — isolated (no runtime), with a reused checker and +//! WARMUP + ITERS iterations. +//! +//! cargo run --release --bin pattern_compare -- [reference.csv] [path.gdb] +//! +//! With no reference CSV it prints the Rust timings and the emptiness verdicts. +//! Given a reference CSV (header +//! `case,expected,got,parse_med_ns,parse_min_ns,check_med_ns,...`) it also prints +//! the per-case ratio and geomean. `got` must match `exp` — a sanity check that +//! the type checker classifies each case the same way as the reference. + +use std::collections::HashMap; +use std::fs; +use std::hint::black_box; +use std::path::Path; +use std::time::Instant; + +use frogql::store::lazy::LazyGraphStore; +use frogql::typing::checker::Typechecker; + +const WARMUP: usize = 1000; +const ITERS: usize = 10000; +const DEFAULT_GDB: &str = "bench/data/ldbc-sf0.1.gdb"; + +fn median_us(mut v: Vec) -> f64 { + v.sort_unstable(); + v[v.len() / 2] as f64 / 1000.0 +} + +/// case -> check_med (us), read from the reference CSV. +fn read_reference_csv(path: &str) -> HashMap { + let mut m = HashMap::new(); + let text = fs::read_to_string(path).unwrap_or_else(|e| panic!("could not read {path}: {e}")); + for line in text.lines().skip(1) { + let f: Vec<&str> = line.split(',').collect(); + if f.len() < 6 { + continue; + } + if let Ok(ns) = f[5].parse::() { + m.insert(f[0].to_string(), ns / 1000.0); + } + } + m +} + +fn main() { + let mut args = std::env::args().skip(1); + let ref_csv = args.next(); + let gdb = args.next().unwrap_or_else(|| DEFAULT_GDB.to_string()); + + let reference: HashMap = ref_csv + .as_deref() + .map(read_reference_csv) + .unwrap_or_default(); + + let lazy = LazyGraphStore::open(Path::new(&gdb)).unwrap_or_else(|e| panic!("open {gdb}: {e}")); + let schema = lazy.catalog().active_schema(); + + // Reduced path-pattern queries, one per internal-bench case (undirected `~` + // edges, query-level WHERE inlined; the IC and aggregate cases are dropped + // because they are not path patterns). i_parse is omitted (a parse error has + // no typecheck to time). + let cases: &[(&str, &str, &str)] = &[ + ("v_label", "valid", "(p: Person)"), + ("v_chain_knows", "valid", "(p: Person)~[:knows]~(f: Person)"), + ("v_where", "valid", "(p: Person WHERE p.id = 933)"), + ( + "v_empty_by_data", + "valid", + "(p: Person WHERE p.id = 1234567890)", + ), + ( + "e_chain4_bad_leaf", + "empty", + "(a: Person)~[:knows]~(b: Person)~[:knows]~(c: Person)~[:knows]~(d: Wagumi)", + ), + ( + "e_chain_mid_bad", + "empty", + "(a: Person)~[:knows]~(b: Wagumi)~[:knows]~(c: Person)", + ), + ( + "e_bad_edge_deep", + "empty", + "(a: Person)~[:knows]~(b: Person)-[:noSuchEdge]->(c: Person)", + ), + ( + "e_ic2_bad_msg", + "empty", + "(p: Person)~[:knows]~(friend: Person)<-[:hasCreator]-(m: Wagumi)", + ), + ( + "e_conflict_label_deep", + "empty", + "(a: Person)~[:knows]~(b: Person)-[:hasCreator]->(b: Comment)", + ), + ( + "e_type_mismatch_chain", + "empty", + "(a: Person)~[:knows]~(b: Person)~[:knows]~(c: Person WHERE c.firstName = 933)", + ), + ( + "e_union_all_bad", + "empty", + "(x: Wagumi)<-[:hasCreator]-(c: Comment) | (x: Wagumi)<-[:hasCreator]-(c: Post)", + ), + ( + "e_repeat_bad_leaf", + "empty", + "(p: Person)~[:knows]~{1,2}(f: Wagumi)", + ), + ("e_label_only", "empty", "(x: Wagumi)"), + ( + "e_type_clash_arith", + "empty", + "(p: Person WHERE p.firstName + p.id > 0)", + ), + ( + "i_unbound_compound_where", + "invalid", + "(p: Person WHERE p.id = 933 AND q.id = 1)", + ), + ( + "i_unbound_in_where_chain", + "invalid", + "(a: Person)~[:knows]~(b: Person)~[:knows]~(c: Person WHERE q.id = 1)", + ), + ]; + + println!( + "\n{:<26}{:>6}{:>9}{:>12}{:>11}{:>10}", + "case", "exp", "got", "rust_med", "ref(us)", "ref/rust" + ); + println!("{}", "-".repeat(76)); + let mut logsum = 0.0f64; + let mut n = 0usize; + let mut mismatches = 0usize; + for (id, expected, q) in cases { + let parsed = frogql::parser::parse_query(q).expect("parse"); + let elaborated = frogql::elaborate::elaborate_query(parsed); + let mut tc = Typechecker::new(schema.clone()); + let r = tc.check_query(&elaborated); + let got = if !r.ok { + "invalid" + } else if r.empty { + "empty" + } else { + "valid" + }; + for _ in 0..WARMUP { + black_box(tc.check_query(black_box(&elaborated))); + } + let mut samples = Vec::with_capacity(ITERS); + for _ in 0..ITERS { + let t = Instant::now(); + black_box(tc.check_query(black_box(&elaborated))); + samples.push(t.elapsed().as_nanos()); + } + let rust_med = median_us(samples); + let bad = if got != *expected { + mismatches += 1; + " <-- VERDICT MISMATCH" + } else { + "" + }; + match reference.get(*id) { + Some(reference_us) => { + let ratio = reference_us / rust_med; + println!( + "{:<26}{:>6}{:>9}{:>12.2}{:>11.2}{:>10.2}{}", + id, expected, got, rust_med, reference_us, ratio, bad + ); + logsum += ratio.ln(); + n += 1; + } + None => println!( + "{:<26}{:>6}{:>9}{:>12.2}{:>11}{:>10}{}", + id, expected, got, rust_med, "-", "-", bad + ), + } + } + println!("{}", "-".repeat(76)); + if n > 0 { + println!( + "geomean ref/rust over {n} cases = {:.2}x (reference slower => rust faster)", + (logsum / n as f64).exp() + ); + } + println!( + "{}", + if mismatches == 0 { + "SANITY OK: rust matched the expected verdict on every case." + } else { + "WARNING: verdict mismatch -- investigate before trusting the timings." + } + ); +} From 30647f3009f278bfe83117950661edc36a742f75 Mon Sep 17 00:00:00 2001 From: Felipe705x Date: Tue, 7 Jul 2026 09:14:48 -0400 Subject: [PATCH 2/2] Decouple typecheck bench: rename to pattern_typecheck, emit own CSV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench now measures only the Rust type checker and writes its own per-case medians to results_rust_typecheck.csv (symmetric to the pygql bench). It no longer reads any reference CSV — comparing against another checker is a separate local join of the two benches' outputs, so neither bench depends on the other's results. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...attern_compare.rs => pattern_typecheck.rs} | 136 ++++++++---------- 1 file changed, 59 insertions(+), 77 deletions(-) rename src/bin/{pattern_compare.rs => pattern_typecheck.rs} (55%) diff --git a/src/bin/pattern_compare.rs b/src/bin/pattern_typecheck.rs similarity index 55% rename from src/bin/pattern_compare.rs rename to src/bin/pattern_typecheck.rs index 9411693..03649f5 100644 --- a/src/bin/pattern_compare.rs +++ b/src/bin/pattern_typecheck.rs @@ -1,23 +1,18 @@ -//! pattern_compare — times the type checker on the reduced LDBC path-pattern -//! cases and, optionally, compares against a reference CSV. +//! pattern_typecheck — times the type checker on the internal-bench cases +//! reduced to the FPPC path-pattern surface (MATCH/RETURN dropped, a +//! query-level WHERE inlined into the descriptor). Isolated from the runtime, +//! with a reused checker and WARMUP + ITERS iterations. //! -//! Each case is an internal-bench query reduced to the FPPC path-pattern surface -//! (MATCH / RETURN dropped, a query-level WHERE inlined into the descriptor). -//! Rust times `check_query` on the reduced pattern — i.e. the path-pattern -//! typecheck only — isolated (no runtime), with a reused checker and -//! WARMUP + ITERS iterations. +//! cargo run --release --bin pattern_typecheck -- [path.gdb] [out.csv] //! -//! cargo run --release --bin pattern_compare -- [reference.csv] [path.gdb] -//! -//! With no reference CSV it prints the Rust timings and the emptiness verdicts. -//! Given a reference CSV (header -//! `case,expected,got,parse_med_ns,parse_min_ns,check_med_ns,...`) it also prints -//! the per-case ratio and geomean. `got` must match `exp` — a sanity check that -//! the type checker classifies each case the same way as the reference. +//! Emits per-case check medians to `results_rust_typecheck.csv` and asserts each +//! verdict (valid / empty / invalid) matches the expected classification. It +//! measures only Rust: any comparison against another checker is a separate step +//! that joins this CSV with the other's — this bench does not read either. -use std::collections::HashMap; -use std::fs; +use std::fs::File; use std::hint::black_box; +use std::io::Write; use std::path::Path; use std::time::Instant; @@ -28,36 +23,17 @@ const WARMUP: usize = 1000; const ITERS: usize = 10000; const DEFAULT_GDB: &str = "bench/data/ldbc-sf0.1.gdb"; -fn median_us(mut v: Vec) -> f64 { +fn median_min_ns(mut v: Vec) -> (u128, u128) { v.sort_unstable(); - v[v.len() / 2] as f64 / 1000.0 -} - -/// case -> check_med (us), read from the reference CSV. -fn read_reference_csv(path: &str) -> HashMap { - let mut m = HashMap::new(); - let text = fs::read_to_string(path).unwrap_or_else(|e| panic!("could not read {path}: {e}")); - for line in text.lines().skip(1) { - let f: Vec<&str> = line.split(',').collect(); - if f.len() < 6 { - continue; - } - if let Ok(ns) = f[5].parse::() { - m.insert(f[0].to_string(), ns / 1000.0); - } - } - m + (v[v.len() / 2], v[0]) } fn main() { let mut args = std::env::args().skip(1); - let ref_csv = args.next(); let gdb = args.next().unwrap_or_else(|| DEFAULT_GDB.to_string()); - - let reference: HashMap = ref_csv - .as_deref() - .map(read_reference_csv) - .unwrap_or_default(); + let out = args + .next() + .unwrap_or_else(|| "results_rust_typecheck.csv".to_string()); let lazy = LazyGraphStore::open(Path::new(&gdb)).unwrap_or_else(|e| panic!("open {gdb}: {e}")); let schema = lazy.catalog().active_schema(); @@ -134,12 +110,11 @@ fn main() { ]; println!( - "\n{:<26}{:>6}{:>9}{:>12}{:>11}{:>10}", - "case", "exp", "got", "rust_med", "ref(us)", "ref/rust" + "\n{:<26}{:>7}{:>9}{:>15}", + "case", "exp", "got", "check_med_us" ); - println!("{}", "-".repeat(76)); - let mut logsum = 0.0f64; - let mut n = 0usize; + println!("{}", "-".repeat(57)); + let mut rows: Vec<(String, String, String, u128, u128, bool)> = Vec::new(); let mut mismatches = 0usize; for (id, expected, q) in cases { let parsed = frogql::parser::parse_query(q).expect("parse"); @@ -162,42 +137,49 @@ fn main() { black_box(tc.check_query(black_box(&elaborated))); samples.push(t.elapsed().as_nanos()); } - let rust_med = median_us(samples); - let bad = if got != *expected { + let (med, min) = median_min_ns(samples); + let ok = got == *expected; + if !ok { mismatches += 1; - " <-- VERDICT MISMATCH" - } else { - "" - }; - match reference.get(*id) { - Some(reference_us) => { - let ratio = reference_us / rust_med; - println!( - "{:<26}{:>6}{:>9}{:>12.2}{:>11.2}{:>10.2}{}", - id, expected, got, rust_med, reference_us, ratio, bad - ); - logsum += ratio.ln(); - n += 1; - } - None => println!( - "{:<26}{:>6}{:>9}{:>12.2}{:>11}{:>10}{}", - id, expected, got, rust_med, "-", "-", bad - ), } + println!( + "{:<26}{:>7}{:>9}{:>15.3}{}", + id, + expected, + got, + med as f64 / 1000.0, + if ok { "" } else { " <-- VERDICT MISMATCH" } + ); + rows.push(( + id.to_string(), + expected.to_string(), + got.to_string(), + med, + min, + ok, + )); + } + println!("{}", "-".repeat(57)); + + let mut f = File::create(&out).unwrap_or_else(|e| panic!("create {out}: {e}")); + writeln!(f, "case,expected,got,check_med_ns,check_min_ns,status").unwrap(); + for (id, exp, got, med, min, ok) in &rows { + writeln!( + f, + "{id},{exp},{got},{med},{min},{}", + if *ok { "PASS" } else { "FAIL" } + ) + .unwrap(); } - println!("{}", "-".repeat(76)); - if n > 0 { + println!("wrote {out}"); + if mismatches == 0 { + println!( + "SANITY OK: rust matched the expected verdict on all {} cases.", + rows.len() + ); + } else { println!( - "geomean ref/rust over {n} cases = {:.2}x (reference slower => rust faster)", - (logsum / n as f64).exp() + "WARNING: {mismatches} verdict mismatch(es) -- investigate before trusting timings." ); } - println!( - "{}", - if mismatches == 0 { - "SANITY OK: rust matched the expected verdict on every case." - } else { - "WARNING: verdict mismatch -- investigate before trusting the timings." - } - ); }