From 1a39bf689fe8e1fc5168a190d4e2ae1c01ec5771 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 17:42:52 +0100 Subject: [PATCH 1/9] Support formula arrays and qualified names; measure the frozen corpus --- .github/workflows/corpus.yml | 99 ++++++++ corpus/README.md | 8 + crates/omasheets-calc/src/lib.rs | 339 ++++++++++++++++++++++++---- crates/omasheets-corpus/src/main.rs | 8 + crates/omasheets-xlsx/src/lib.rs | 36 +++ docs/FUNCTIONS.md | 11 +- 6 files changed, 449 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/corpus.yml diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml new file mode 100644 index 0000000..963eb48 --- /dev/null +++ b/.github/workflows/corpus.yml @@ -0,0 +1,99 @@ +name: Frozen formula corpus + +on: + workflow_dispatch: + inputs: + baseline: + description: Baseline commit or branch to compare on the same runner + required: true + default: main + pull_request: + paths: + - .github/workflows/corpus.yml + +permissions: + contents: read + +concurrency: + group: corpus-${{ github.ref }} + cancel-in-progress: true + +jobs: + compare: + name: Frozen Enron before and after + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CARGO_INCREMENTAL: "0" + CARGO_TARGET_DIR: ${{ runner.temp }}/omasheets-corpus-target + CORPUS_EVIDENCE: ${{ runner.temp }}/omasheets-corpus-evidence + CORPUS_DATA: ${{ runner.temp }}/omasheets-corpus-data + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: candidate + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha || inputs.baseline }} + path: baseline + persist-credentials: false + - name: Install the minimum toolchain and archive extractor + run: | + sudo apt-get update + sudo apt-get install --yes p7zip-full + rustup toolchain install 1.88.0 --profile minimal --no-self-update + rustup default 1.88.0 + - name: Fetch the registered archive and preserve the frozen denominator + working-directory: candidate + run: | + cmp corpus/sources/enron-figshare.jsonl ../baseline/corpus/sources/enron-figshare.jsonl + cmp corpus/sources/enron-figshare.json ../baseline/corpus/sources/enron-figshare.json + python scripts/fetch_corpus.py corpus/sources/enron-figshare.json "$CORPUS_DATA" + mkdir -p "$CORPUS_EVIDENCE/before" "$CORPUS_EVIDENCE/after" + - name: Build the baseline scorer + working-directory: baseline + run: cargo build --locked --release -p omasheets-corpus + - name: Verify and score the baseline alone + working-directory: candidate + run: | + "$CARGO_TARGET_DIR/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json" + python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CARGO_TARGET_DIR/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30 + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" + - name: Build the candidate scorer + working-directory: candidate + run: cargo build --locked --release -p omasheets-corpus + - name: Score the candidate alone and report aggregate changes + working-directory: candidate + run: | + python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CARGO_TARGET_DIR/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30 + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path(os.environ["CORPUS_EVIDENCE"]) + before = json.loads((root / "before/summary.json").read_text())["owned_summary"] + after = json.loads((root / "after/summary.json").read_text())["owned_summary"] + keys = ["opened", "formula_cells_observed", "formula_cells_loaded", "formula_cells_compared", "stored_values_matched", "stored_values_mismatched", "formula_parse_rate", "stored_value_match_rate"] + lines = ["| Metric | Before | After |", "|---|---:|---:|"] + lines.extend(f"| {key} | {before[key]} | {after[key]} |" for key in keys) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as output: + output.write("\n".join(lines) + "\n") + assert after["opened"] == before["opened"], "Workbook-open denominator changed" + assert after["formula_cells_observed"] == before["formula_cells_observed"], "Formula denominator changed" + assert after["formula_cells_loaded"] >= before["formula_cells_loaded"], "Formula compilation regressed" + assert after["stored_values_matched"] >= before["stored_values_matched"], "Stored-value matches regressed" + PY + - name: Upload aggregate evidence only + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: frozen-formula-corpus + if-no-files-found: warn + path: | + ${{ runner.temp }}/omasheets-corpus-evidence/before/summary.json + ${{ runner.temp }}/omasheets-corpus-evidence/after/summary.json + ${{ runner.temp }}/omasheets-corpus-evidence/after/delta.json diff --git a/corpus/README.md b/corpus/README.md index 54288ce..2cf2c90 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -124,6 +124,14 @@ access terms. See `corpus/sources/README.md` for what is registered. ## Corpus policy +The **Frozen formula corpus** Actions workflow scores a baseline and candidate +sequentially on the same hosted runner, using the registered archive and the +unchanged 1,000-workbook manifest. Run it manually with a baseline revision; +changes to the workflow itself also exercise it in a pull request. Only +aggregate before/after summaries are uploaded. Syntax failures include a fixed +token-class histogram to guide parser work without exposing formula contents. +This is corpus compatibility evidence, not target-desktop performance evidence. + - Record the upstream corpus name, retrieval date, license or access terms, and sampling method beside every frozen manifest. - Do not commit source workbooks, extracted cell contents, local paths, or model diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index fe287f0..10e13fa 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -191,6 +191,8 @@ enum Expr { Error(CalcError), /// An omitted argument, as in `IF(x,,y)`; evaluates to blank. Empty, + /// A bounded rectangular array of literal values, with no dependencies. + Array(ArrayValue), Reference(R), UnaryMinus(Box>), Percent(Box>), @@ -548,6 +550,7 @@ fn visit_references(expression: &Expr, visit: &mut impl FnMut(CellId)) { | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -594,6 +597,7 @@ fn rebind_references(expression: &mut Expr, map: &mut impl FnMut(CellId) | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -1019,13 +1023,6 @@ impl Workbook { statistics } - fn range_len(&self, node: usize) -> usize { - match self.range_shape(node) { - RangeShape::Rectangle { rows, columns, .. } => rows * columns, - RangeShape::Members { .. } => self.cells[node].dependencies.len(), - } - } - /// The cell at a row-major position inside the range, if it exists. fn range_cell(&self, node: usize, index: usize) -> Option { match self.range_shape(node) { @@ -1378,6 +1375,7 @@ impl Workbook { Expr::Text(value) => Value::Text(value.clone()), Expr::Error(error) => Value::Error(error.clone()), Expr::Empty => Value::Blank, + Expr::Array(array) => array.values[0].clone(), Expr::Reference(index) => self.cells[*index].value.clone(), Expr::UnaryMinus(inner) => match self.evaluate(inner) { Value::Number(value) => number_value(-value), @@ -2088,39 +2086,40 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { + let Some(input) = ArrayInput::new(&arguments[1]) else { return Value::Error(CalcError::InvalidArguments); }; + let (rows, columns) = input.shape(); let (candidates, results): (Vec, Vec) = if arguments.len() == 3 { - let Some((result_node, result_rows, result_columns)) = range_parts(&arguments[2]) - else { + let Some(result_input) = ArrayInput::new(&arguments[2]) else { return Value::Error(CalcError::InvalidArguments); }; + let (result_rows, result_columns) = result_input.shape(); if (rows != 1 && columns != 1) || (result_rows != 1 && result_columns != 1) || rows * columns != result_rows * result_columns { return Value::Error(CalcError::InvalidArguments); } - (self.range_values(node), self.range_values(result_node)) + (input.values(self), result_input.values(self)) } else if rows == 1 || columns == 1 { - let values = self.range_values(node); + let values = input.values(self); (values.clone(), values) } else if columns > rows { // More columns than rows: search the first row, answer from the last. let candidates = (0..columns) - .map(|column| self.range_value(node, column)) + .map(|column| input.value(self, column)) .collect(); let results = (0..columns) - .map(|column| self.range_value(node, (rows - 1) * columns + column)) + .map(|column| input.value(self, (rows - 1) * columns + column)) .collect(); (candidates, results) } else { let candidates = (0..rows) - .map(|row| self.range_value(node, row * columns)) + .map(|row| input.value(self, row * columns)) .collect(); let results = (0..rows) - .map(|row| self.range_value(node, row * columns + columns - 1)) + .map(|row| input.value(self, row * columns + columns - 1)) .collect(); (candidates, results) }; @@ -2333,6 +2332,7 @@ impl Workbook { /// `#N/A`. Bounded by [`MAX_RANGE_CELLS`]. fn evaluate_array(&self, expression: &Expr) -> Result { match expression { + Expr::Array(array) => Ok(array.clone()), Expr::RangeNode { node, rows, @@ -2502,9 +2502,10 @@ impl Workbook { fn evaluate_lookup_function(&self, function: Function, arguments: &[Expr]) -> Value { match function { Function::Index if matches!(arguments.len(), 2 | 3) => { - let Some((node, rows, columns)) = range_parts(&arguments[0]) else { + let Some(input) = ArrayInput::new(&arguments[0]) else { return Value::Error(CalcError::InvalidArguments); }; + let (rows, columns) = input.shape(); // An error row (a failed MATCH) is the result, as in Excel. let row = match positive_index(self.evaluate(&arguments[1])) { Ok(row) => row, @@ -2519,7 +2520,7 @@ impl Workbook { 1 } else if rows == 1 { return if row <= columns { - self.range_value(node, row - 1) + input.value(self, row - 1) } else { Value::Error(CalcError::InvalidReference) }; @@ -2529,7 +2530,7 @@ impl Workbook { if row > rows || column > columns { return Value::Error(CalcError::InvalidReference); } - self.range_value(node, (row - 1) * columns + column - 1) + input.value(self, (row - 1) * columns + column - 1) } Function::Match if matches!(arguments.len(), 2 | 3) => { let mode = if arguments.len() == 3 { @@ -2544,13 +2545,14 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { + let Some(input) = ArrayInput::new(&arguments[1]) else { return Value::Error(CalcError::InvalidArguments); }; + let (rows, columns) = input.shape(); if rows != 1 && columns != 1 { return Value::Error(CalcError::InvalidArguments); } - let candidates = self.range_values(node); + let candidates = input.values(self); match match_position(&lookup, &candidates, mode) { Ok(position) => Value::Number((position + 1) as f64), Err(error) => Value::Error(error), @@ -2577,9 +2579,10 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { + let Some(input) = ArrayInput::new(&arguments[1]) else { return Value::Error(CalcError::InvalidArguments); }; + let (rows, columns) = input.shape(); let Ok(offset) = positive_index(self.evaluate(&arguments[2])) else { return Value::Error(CalcError::InvalidValue); }; @@ -2594,9 +2597,9 @@ impl Workbook { } let at = |lane: usize, position: usize| { if vertical { - self.range_value(node, lane * columns + position) + input.value(self, lane * columns + position) } else { - self.range_value(node, position * columns + lane) + input.value(self, position * columns + lane) } }; let candidates: Vec = (0..lanes).map(|lane| at(lane, 0)).collect(); @@ -2610,25 +2613,25 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((lookup_node, lookup_rows, lookup_columns)) = range_parts(&arguments[1]) - else { + let Some(lookup_input) = ArrayInput::new(&arguments[1]) else { return Value::Error(CalcError::InvalidArguments); }; - let Some((return_node, return_rows, return_columns)) = range_parts(&arguments[2]) - else { + let Some(return_input) = ArrayInput::new(&arguments[2]) else { return Value::Error(CalcError::InvalidArguments); }; - let length = self.range_len(lookup_node); + let (lookup_rows, lookup_columns) = lookup_input.shape(); + let (return_rows, return_columns) = return_input.shape(); + let length = lookup_rows * lookup_columns; if (lookup_rows != 1 && lookup_columns != 1) || (return_rows != 1 && return_columns != 1) - || length != self.range_len(return_node) + || length != return_rows * return_columns { return Value::Error(CalcError::InvalidArguments); } for index in 0..length { - let candidate = self.range_value(lookup_node, index); + let candidate = lookup_input.value(self, index); if lookup_equal(&lookup, &candidate) { - return self.range_value(return_node, index); + return return_input.value(self, index); } } if arguments.len() == 4 { @@ -2755,12 +2758,62 @@ fn typed_compare(left: &Value, right: &Value) -> Result, } +/// Lookup inputs borrow constants and retain the existing range fast path. +enum ArrayInput<'a> { + Range { + node: usize, + rows: usize, + columns: usize, + }, + Constant(&'a ArrayValue), +} + +impl<'a> ArrayInput<'a> { + fn new(expression: &'a Expr) -> Option { + match expression { + Expr::RangeNode { + node, + rows, + columns, + } => Some(Self::Range { + node: *node, + rows: *rows, + columns: *columns, + }), + Expr::Array(array) => Some(Self::Constant(array)), + _ => None, + } + } + + fn shape(&self) -> (usize, usize) { + match self { + Self::Range { rows, columns, .. } => (*rows, *columns), + Self::Constant(array) => array.shape(), + } + } + + fn value(&self, workbook: &Workbook, index: usize) -> Value { + match self { + Self::Range { node, .. } => workbook.range_value(*node, index), + Self::Constant(array) => array.values[index].clone(), + } + } + + fn values(&self, workbook: &Workbook) -> Vec { + match self { + Self::Range { node, .. } => workbook.range_values(*node), + Self::Constant(array) => array.values.clone(), + } + } +} + impl ArrayValue { fn scalar(value: Value) -> Self { Self { @@ -2802,7 +2855,7 @@ fn broadcast_shape(shapes: &[(usize, usize)]) -> Result<(usize, usize), CalcErro /// lookups, criteria functions) produce one value and stop the search. fn contains_array_operand(expression: &Expr) -> bool { match expression { - Expr::RangeNode { .. } => true, + Expr::RangeNode { .. } | Expr::Array(_) => true, Expr::UnaryMinus(inner) | Expr::Percent(inner) => contains_array_operand(inner), Expr::Binary(_, left, right) => { contains_array_operand(left) || contains_array_operand(right) @@ -2885,17 +2938,6 @@ fn literal(value: Value) -> Expr { } } -fn range_parts(expression: &Expr) -> Option<(usize, usize, usize)> { - match expression { - Expr::RangeNode { - node, - rows, - columns, - } => Some((*node, *rows, *columns)), - _ => None, - } -} - fn positive_index(value: Value) -> Result { let value = number(value)?; if value >= 1.0 && value.fract() == 0.0 && value <= usize::MAX as f64 { @@ -3800,6 +3842,7 @@ fn compile_expression( Expr::Text(value) => Expr::Text(value), Expr::Error(error) => Expr::Error(error), Expr::Empty => Expr::Empty, + Expr::Array(array) => Expr::Array(array), Expr::Reference(cell) => Expr::Reference(indices[&cell]), Expr::UnaryMinus(inner) => { Expr::UnaryMinus(Box::new(compile_expression(*inner, indices, range_nodes))) @@ -3839,6 +3882,7 @@ fn count_nodes(expression: &Expr) -> usize { Expr::Binary(_, left, right) => count_nodes(left) + count_nodes(right), Expr::Function(_, arguments) => arguments.iter().map(count_nodes).sum(), Expr::Range { members, .. } => members.as_ref().map_or(0, Vec::len), + Expr::Array(array) => array.values.len(), _ => 0, } } @@ -3875,6 +3919,7 @@ fn collect_dependencies( | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -4088,6 +4133,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { Ok(expression) } Some(b'"') => self.parse_string(), + Some(b'{') => self.parse_array_constant(), Some(b'\'') => self.parse_quoted_sheet_reference(), Some(b'#') => self.parse_error_literal(), Some(b'[') @@ -4110,6 +4156,80 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { self.remaining().chars().take(64).collect() } + fn parse_array_constant(&mut self) -> Result { + self.expect(b'{')?; + let mut values = Vec::new(); + let mut rows = 0; + let mut columns = None; + let mut row_columns = 0; + loop { + self.skip_space(); + let start = self.offset; + let value = match self.peek() { + Some(b'"') => self.parse_string()?, + Some(b'#') => self.parse_error_literal()?, + Some(b'+') | Some(b'-') => { + let negative = self.peek() == Some(b'-'); + self.offset += 1; + self.skip_space(); + let Expr::Number(number) = self.parse_number()? else { + unreachable!("number parser returns a number") + }; + Expr::Number(if negative { -number } else { number }) + } + Some(byte) if byte.is_ascii_digit() || byte == b'.' => self.parse_number()?, + Some(byte) if byte.is_ascii_alphabetic() => { + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphabetic()) { + self.offset += 1; + } + match self.source[start..self.offset] + .to_ascii_uppercase() + .as_str() + { + "TRUE" => Expr::Boolean(true), + "FALSE" => Expr::Boolean(false), + _ => return Err(FormulaError::UnexpectedToken(start)), + } + } + _ => return Err(FormulaError::UnexpectedToken(start)), + }; + let value = match value { + Expr::Number(value) if value.is_finite() => Value::Number(value), + Expr::Boolean(value) => Value::Boolean(value), + Expr::Text(value) => Value::Text(value), + Expr::Error(value) => Value::Error(value), + _ => return Err(FormulaError::UnexpectedToken(start)), + }; + if values.len() == MAX_RANGE_CELLS { + return Err(FormulaError::RangeTooLarge); + } + values.push(value); + row_columns += 1; + self.skip_space(); + match self.peek() { + Some(b',') => self.offset += 1, + Some(b';') | Some(b'}') => { + if columns.is_some_and(|width| width != row_columns) { + return Err(FormulaError::UnexpectedToken(self.offset)); + } + columns = Some(row_columns); + rows += 1; + row_columns = 0; + let finished = self.peek() == Some(b'}'); + self.offset += 1; + if finished { + return Ok(Expr::Array(ArrayValue { + rows, + columns: columns.expect("completed row"), + values, + })); + } + } + _ => return Err(FormulaError::UnexpectedToken(self.offset)), + } + } + } + fn parse_error_literal(&mut self) -> Result { let start = self.offset; while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || matches!(byte, b'#' | b'/' | b'!' | b'?')) @@ -4205,8 +4325,16 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { /// same sheet table and a bounded depth for names that use names. A name /// scoped to the formula's sheet wins over a workbook-level one. fn parse_defined_name(&mut self, token: &str) -> Result { + self.parse_defined_name_in_sheet(token, self.sheet) + } + + fn parse_defined_name_in_sheet( + &mut self, + token: &str, + sheet: u32, + ) -> Result { let name = token.trim_start_matches('$'); - let Some(definition) = self.defined_names.resolve(self.sheet, &name.to_lowercase()) else { + let Some(definition) = self.defined_names.resolve(sheet, &name.to_lowercase()) else { return Err(FormulaError::UnknownName(name.into())); }; if self.name_depth >= MAX_NAME_DEPTH { @@ -4214,7 +4342,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { } let mut inner = Parser::new_structured( definition, - self.sheet, + sheet, self.sheet_names, self.defined_names, self.structured, @@ -4305,11 +4433,23 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { return Ok(Expr::Error(CalcError::InvalidReference)); } let start = self.offset; - while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'$') { + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || matches!(byte, b'$' | b'_' | b'.')) + { self.offset += 1; } - let first = parse_a1(&self.source[start..self.offset], sheet)?; - self.parse_range_tail(first, sheet) + let token = &self.source[start..self.offset]; + match parse_a1(token, sheet) { + Ok(first) => self.parse_range_tail(first, sheet), + Err(_) + if self + .defined_names + .resolve(sheet, &token.trim_start_matches('$').to_lowercase()) + .is_some() => + { + self.parse_defined_name_in_sheet(token, sheet) + } + Err(error) => Err(error), + } } fn parse_range_tail(&mut self, first: CellId, sheet: u32) -> Result { @@ -4498,9 +4638,11 @@ pub fn supported_function_names() -> impl Iterator { fn parse_function_name(name: &str) -> Result { let upper = name.to_ascii_uppercase(); + let registered = upper.strip_prefix("_XLFN.").unwrap_or(&upper); + let registered = registered.strip_prefix("_XLWS.").unwrap_or(registered); FUNCTION_REGISTRY .iter() - .find(|(candidate, _)| *candidate == upper) + .find(|(candidate, _)| *candidate == registered) .map(|(_, function)| *function) .ok_or(FormulaError::UnsupportedFunction(upper)) } @@ -4666,6 +4808,103 @@ fn expand_range(first: CellId, second: CellId) -> Result { mod tests { use super::*; + #[test] + fn array_constants_feed_aggregates_broadcasting_and_lookups() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 10.0); + workbook.set_number(cell(1, 0), 20.0); + let examples = [ + ("=SUM({1,2;3,4})", Value::Number(10.0)), + ("=SUMPRODUCT(A1:A2,{2;3})", Value::Number(80.0)), + ("=SUM(IF({TRUE,FALSE},A1,2))", Value::Number(12.0)), + ("=SUM({1,2}+{10;20})", Value::Number(66.0)), + ("=INDEX({1,2;3,4},2,1)", Value::Number(3.0)), + ("=INDEX({1,2,3},2)", Value::Number(2.0)), + ("=MATCH(20,{10;20;30},0)", Value::Number(2.0)), + ("=VLOOKUP(2,{1,10;2,20},2,FALSE)", Value::Number(20.0)), + ("=HLOOKUP(2,{1,2;10,20},2,FALSE)", Value::Number(20.0)), + ("=XLOOKUP(2,{1,2},{10,20})", Value::Number(20.0)), + ("=LOOKUP(2,{1,2,3},{10,20,30})", Value::Number(20.0)), + ("=SUM({-2,+3,1e2})", Value::Number(101.0)), + ("=SUM({1,#N/A})", Value::Error(CalcError::NotAvailable)), + ( + "=TEXTJOIN(\"/\",TRUE,{\"a\",\"\";\"b\",\"c\"})", + Value::Text("a/b/c".into()), + ), + ("=SUM(_xlfn.IFERROR({1,#N/A},0))", Value::Number(1.0)), + ]; + for (formula, expected) in examples { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 3)), expected, "{formula}"); + } + workbook.define_name("weights", "{2;3}"); + workbook + .set_formula(cell(0, 3), "=SUMPRODUCT(A1:A2,weights)") + .unwrap(); + let changed = workbook.set_number(cell(1, 0), 30.0); + assert!(changed.evaluated.contains(&cell(0, 3))); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(110.0)); + } + + #[test] + fn malformed_array_constants_are_rejected_atomically() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 17.0); + for formula in [ + "={}", + "={1,}", + "={;1}", + "={1;}", + "={1,2;3}", + "={1,A1}", + "={SUM(1)}", + "={{1}}", + "={1+2}", + "={1%}", + "={1e999}", + "={TRUEFALSE}", + "={1;2,3}", + ] { + assert!( + workbook.set_formula(cell(0, 0), formula).is_err(), + "{formula}" + ); + assert_eq!(workbook.value(cell(0, 0)), Value::Number(17.0)); + } + let oversized = format!("={{{}0}}", "0,".repeat(MAX_RANGE_CELLS)); + assert_eq!( + workbook.set_formula(cell(0, 0), &oversized), + Err(FormulaError::RangeTooLarge) + ); + assert_eq!(workbook.value(cell(0, 0)), Value::Number(17.0)); + } + + #[test] + fn qualified_names_bind_to_the_named_sheet_and_recalculate() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Input"); + workbook.define_sheet(1, "Other sheet"); + workbook.define_sheet_name(0, "rate.total", "A1*2"); + workbook.define_sheet_name(1, "rate.total", "A1*3"); + workbook.set_number(CellId::new(0, 0, 0), 10.0); + workbook.set_number(CellId::new(1, 0, 0), 20.0); + workbook + .set_formula(cell(0, 1), "=Input!rate.total+'Other sheet'!rate.total") + .unwrap(); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(80.0)); + let changed = workbook.set_number(CellId::new(1, 0, 0), 30.0); + assert!(changed.evaluated.contains(&cell(0, 1))); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(110.0)); + assert!(matches!( + workbook.set_formula(cell(0, 1), "=_xlfn.MISSING(1)"), + Err(FormulaError::UnsupportedFunction(_)) + )); + assert!(matches!( + workbook.set_formula(cell(0, 1), "='[1]Other sheet'!rate.total"), + Err(FormulaError::ExternalReference(_)) + )); + } + fn cell(row: u32, column: u32) -> CellId { CellId::new(0, row, column) } diff --git a/crates/omasheets-corpus/src/main.rs b/crates/omasheets-corpus/src/main.rs index bb19c24..4dadbff 100644 --- a/crates/omasheets-corpus/src/main.rs +++ b/crates/omasheets-corpus/src/main.rs @@ -128,6 +128,7 @@ struct OwnedSummary { /// and the number of workbooks naming each, bounded like the per-file map. unsupported_functions: BTreeMap, unsupported_reasons: BTreeMap, + syntax_failure_tokens: BTreeMap, /// Workbooks the importer opened only after skipping sheet entries that /// have no worksheet part, and how many such entries it skipped in all. workbooks_with_skipped_sheets: u64, @@ -815,6 +816,12 @@ fn accumulate_owned(summary: &mut OwnedSummary, owned: &OwnedProbe) { .entry(reason.clone()) .or_default() += *cells as u64; } + for (token, cells) in &report.syntax_failure_tokens { + *summary + .syntax_failure_tokens + .entry(token.clone()) + .or_default() += *cells as u64; + } if !report.skipped_sheets.is_empty() { summary.workbooks_with_skipped_sheets += 1; summary.skipped_sheets += report.skipped_sheets.len() as u64; @@ -1071,6 +1078,7 @@ mod tests { "unsupported_function".to_string(), 10 - loaded, )]), + syntax_failure_tokens: BTreeMap::new(), skipped_sheets: if loaded == 8 { vec!["Module1".to_string(), "Module2".to_string()] } else { diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index 5db96c2..659894c 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -111,6 +111,9 @@ pub struct ScoreReport { pub unsupported_functions: BTreeMap, /// Compile-failure kinds and how many formula cells hit each. pub unsupported_reasons: BTreeMap, + /// Syntax failures grouped by a fixed token class, never formula text. + #[serde(default)] + pub syntax_failure_tokens: BTreeMap, /// Sheet entries without a worksheet part that the importer skipped. #[serde(default)] pub skipped_sheets: Vec, @@ -214,6 +217,38 @@ impl ImportedWorkbook { pub fn report(&self) -> ScoreReport { let parity = self.parity(); + let mut syntax_failure_tokens = BTreeMap::new(); + for failure in &self.unsupported { + let FormulaError::UnexpectedToken(offset) = failure.error else { + continue; + }; + let Ok(index) = self + .source_cells + .binary_search_by_key(&failure.cell, |cell| cell.cell) + else { + continue; + }; + let Some(source) = self.source_cells[index].formula.as_deref() else { + continue; + }; + let source = source.strip_prefix('=').unwrap_or(source); + let token = match source.as_bytes().get(offset) { + Some(b'{') | Some(b'}') => "array_brace", + Some(b':') => "range_colon", + Some(b',') => "comma", + Some(b';') => "semicolon", + Some(b'!') => "sheet_separator", + Some(b'[') | Some(b']') => "square_bracket", + Some(b'@') => "implicit_intersection", + Some(b'\\') => "backslash_identifier", + Some(b'(') | Some(b')') => "parenthesis", + Some(byte) if !byte.is_ascii() => "non_ascii_identifier", + Some(byte) if byte.is_ascii_alphabetic() => "identifier", + None => "end_of_formula", + _ => "other", + }; + *syntax_failure_tokens.entry(token.to_string()).or_insert(0) += 1; + } ScoreReport { schema: 2, engine: ENGINE_NAME.into(), @@ -228,6 +263,7 @@ impl ImportedWorkbook { unsupported_formulas: parity.unsupported_formulas, unsupported_functions: self.unsupported_functions(), unsupported_reasons: self.unsupported_reasons(), + syntax_failure_tokens, skipped_sheets: self.skipped_sheets.clone(), } } diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 109e3df..4167db0 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -9,14 +9,21 @@ regenerate this list. Operators: `+ - * / ^ & %`, unary `+`/`-`, comparisons `= <> < <= > >=`, error literals (`#REF!`, `#N/A`, `#DIV/0!`, `#VALUE!`, `#NUM!`, `#NAME?`, `#NULL!`, and `Sheet!#REF!` for a deleted cell on another sheet), omitted arguments, bounded rectangular ranges, absolute markers, -cross-sheet references, workbook and sheet-scoped defined names (tokens past +cross-sheet references, workbook and sheet-scoped defined names (including +`Sheet!LocalName`; tokens past the grid such as `Table1` are names), implicit intersection of a range in scalar position, and elementwise evaluation of range expressions inside aggregate arguments (`SUM(IF(A1:A5=0,0,B1:B5))`, `SUMPRODUCT((A1:A5>2)*B1:B5)`). +Rectangular array constants support numbers, text, booleans and error literals, +comma-separated columns and semicolon-separated rows, up to 1,000,000 values. +They work in aggregates, elementwise expressions and INDEX/MATCH/LOOKUP, +VLOOKUP/HLOOKUP/XLOOKUP. A scalar use takes the first value; spilling into +neighbouring cells is not implemented. `_xlfn.` and `_xlfn._xlws.` prefixes +resolve only to functions already in the registry. Deliberately unsupported: `TODAY`, `NOW`, `RAND` and every other volatile function (until the calculation context consumes stored tick events), external workbook references, -3D references, array constants and array formulas, `INDIRECT`, `OFFSET`, +3D references, spilling array formulas, `INDIRECT`, `OFFSET`, `CELL`, add-in (`_xll.`) calls, locale-sensitive parsing such as `DATEVALUE` and `TEXT`, and the 1904 date system. From db0d27d098ebe0b9414e52a4a5e2661e81964524 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 17:45:33 +0100 Subject: [PATCH 2/9] Initialize corpus paths after the runner starts --- .github/workflows/corpus.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 963eb48..46fd6e3 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -25,10 +25,12 @@ jobs: timeout-minutes: 45 env: CARGO_INCREMENTAL: "0" - CARGO_TARGET_DIR: ${{ runner.temp }}/omasheets-corpus-target - CORPUS_EVIDENCE: ${{ runner.temp }}/omasheets-corpus-evidence - CORPUS_DATA: ${{ runner.temp }}/omasheets-corpus-data steps: + - name: Set runner-local evidence directories + run: | + echo "CARGO_TARGET_DIR=$RUNNER_TEMP/omasheets-corpus-target" >> "$GITHUB_ENV" + echo "CORPUS_EVIDENCE=$RUNNER_TEMP/omasheets-corpus-evidence" >> "$GITHUB_ENV" + echo "CORPUS_DATA=$RUNNER_TEMP/omasheets-corpus-data" >> "$GITHUB_ENV" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} From 12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 18:00:32 +0100 Subject: [PATCH 3/9] Add financial and statistical formulas and general range endpoints --- .github/workflows/corpus.yml | 23 +- corpus/README.md | 3 +- crates/omasheets-calc/src/lib.rs | 374 ++++++++++++++++-- .../tests/native_controls.rs | 30 ++ crates/omasheets-xlsx/src/lib.rs | 37 ++ docs/FUNCTIONS.md | 14 +- docs/NATIVE-SPREADSHEET-SUPPORT.md | 4 +- 7 files changed, 446 insertions(+), 39 deletions(-) diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 46fd6e3..28e7b3d 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Set runner-local evidence directories run: | - echo "CARGO_TARGET_DIR=$RUNNER_TEMP/omasheets-corpus-target" >> "$GITHUB_ENV" + echo "CORPUS_TARGETS=$RUNNER_TEMP/omasheets-corpus-targets" >> "$GITHUB_ENV" echo "CORPUS_EVIDENCE=$RUNNER_TEMP/omasheets-corpus-evidence" >> "$GITHUB_ENV" echo "CORPUS_DATA=$RUNNER_TEMP/omasheets-corpus-data" >> "$GITHUB_ENV" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -56,20 +56,20 @@ jobs: mkdir -p "$CORPUS_EVIDENCE/before" "$CORPUS_EVIDENCE/after" - name: Build the baseline scorer working-directory: baseline - run: cargo build --locked --release -p omasheets-corpus + run: cargo build --locked --release -p omasheets-corpus --target-dir "$CORPUS_TARGETS/baseline" - name: Verify and score the baseline alone working-directory: candidate run: | - "$CARGO_TARGET_DIR/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json" - python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CARGO_TARGET_DIR/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30 + "$CORPUS_TARGETS/baseline/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json" + python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CORPUS_TARGETS/baseline/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30 python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" - name: Build the candidate scorer working-directory: candidate - run: cargo build --locked --release -p omasheets-corpus + run: cargo build --locked --release -p omasheets-corpus --target-dir "$CORPUS_TARGETS/candidate" - name: Score the candidate alone and report aggregate changes working-directory: candidate run: | - python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CARGO_TARGET_DIR/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30 + python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CORPUS_TARGETS/candidate/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30 python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" python - <<'PY' import json @@ -77,8 +77,15 @@ jobs: from pathlib import Path root = Path(os.environ["CORPUS_EVIDENCE"]) - before = json.loads((root / "before/summary.json").read_text())["owned_summary"] - after = json.loads((root / "after/summary.json").read_text())["owned_summary"] + before_report = json.loads((root / "before/summary.json").read_text()) + before = before_report["owned_summary"] + after_report = json.loads((root / "after/summary.json").read_text()) + after = after_report["owned_summary"] + # Retain aggregate evidence in logs if artifact storage is unavailable. + print("FROZEN_CORPUS_BEFORE " + json.dumps(before_report, sort_keys=True)) + print("FROZEN_CORPUS_AFTER " + json.dumps(after_report, sort_keys=True)) + print("FROZEN_CORPUS_DELTA " + (root / "after/delta.json").read_text().replace("\n", " ")) + assert "syntax_failure_tokens" in after, "Candidate scorer lacks the new diagnostic schema" keys = ["opened", "formula_cells_observed", "formula_cells_loaded", "formula_cells_compared", "stored_values_matched", "stored_values_mismatched", "formula_parse_rate", "stored_value_match_rate"] lines = ["| Metric | Before | After |", "|---|---:|---:|"] lines.extend(f"| {key} | {before[key]} | {after[key]} |" for key in keys) diff --git a/corpus/README.md b/corpus/README.md index 2cf2c90..a61d4af 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -125,7 +125,8 @@ access terms. See `corpus/sources/README.md` for what is registered. ## Corpus policy The **Frozen formula corpus** Actions workflow scores a baseline and candidate -sequentially on the same hosted runner, using the registered archive and the +sequentially on the same hosted runner, in separate build directories, using +the registered archive and the unchanged 1,000-workbook manifest. Run it manually with a baseline revision; changes to the workflow itself also exercise it in a pull request. Only aggregate before/after summaries are uploaded. Syntax failures include a fixed diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index 10e13fa..4645f72 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -361,10 +361,16 @@ enum Function { WorkDay, Lookup, Pmt, + Pv, + Irr, Npv, Xnpv, Xirr, NormDist, + NormSDist, + NormSDistLegacy, + CovarianceP, + CovarianceS, AverageA, Correl, IsBlank, @@ -1595,12 +1601,20 @@ impl Workbook { } if matches!( function, - Function::Pmt | Function::Npv | Function::Xnpv | Function::Xirr + Function::Pmt + | Function::Pv + | Function::Irr + | Function::Npv + | Function::Xnpv + | Function::Xirr ) { return self.evaluate_financial_function(function, arguments); } - if function == Function::Correl { - return self.evaluate_correl(arguments); + if matches!( + function, + Function::Correl | Function::CovarianceP | Function::CovarianceS + ) { + return self.evaluate_paired_statistics(function, arguments); } if matches!( @@ -1637,6 +1651,22 @@ impl Workbook { match function { Function::Median if !numbers.is_empty() => median(numbers), Function::NormDist => normal_distribution(&values), + Function::NormSDist | Function::NormSDistLegacy => { + let required = if function == Function::NormSDistLegacy { + 1 + } else { + 2 + }; + if values.len() != required { + return Value::Error(CalcError::InvalidArguments); + } + normal_distribution(&[ + values[0].clone(), + Value::Number(0.0), + Value::Number(1.0), + values.get(1).cloned().unwrap_or(Value::Boolean(true)), + ]) + } Function::AverageA => average_a(&values), Function::StDev => deviation(&numbers, true, true), Function::StDevP => deviation(&numbers, false, true), @@ -1735,10 +1765,14 @@ impl Workbook { | Function::NetworkDays | Function::WorkDay | Function::Pmt + | Function::Pv + | Function::Irr | Function::Npv | Function::Xnpv | Function::Xirr | Function::Correl + | Function::CovarianceP + | Function::CovarianceS | Function::Date | Function::Year | Function::Month @@ -2129,10 +2163,10 @@ impl Workbook { } } - /// `PMT`, `NPV`, `XNPV` and `XIRR`. + /// Present values, annuity payments, and periodic or dated returns. fn evaluate_financial_function(&self, function: Function, arguments: &[Expr]) -> Value { let result = match function { - Function::Pmt => { + Function::Pmt | Function::Pv => { if !matches!(arguments.len(), 3..=5) { return Value::Error(CalcError::InvalidArguments); } @@ -2143,7 +2177,12 @@ impl Workbook { Err(error) => return Value::Error(error), } } - payment( + let calculate = if function == Function::Pv { + present_value + } else { + payment + }; + calculate( numbers[0], numbers[1], numbers[2], @@ -2151,6 +2190,22 @@ impl Workbook { numbers.get(4).copied().unwrap_or(0.0) != 0.0, ) } + Function::Irr => { + if !matches!(arguments.len(), 1 | 2) { + return Value::Error(CalcError::InvalidArguments); + } + let mut values = Vec::new(); + self.flatten_values(&arguments[0], &mut values); + if let Some(error) = first_error(&values) { + return Value::Error(error); + } + let guess = arguments + .get(1) + .map_or(Ok(0.1), |argument| number(self.evaluate(argument))); + guess.and_then(|guess| { + periodic_internal_rate_of_return(&numeric_only(&values), guess) + }) + } Function::Npv => { if arguments.len() < 2 { return Value::Error(CalcError::InvalidArguments); @@ -2208,9 +2263,8 @@ impl Workbook { } } - /// `CORREL(array1, array2)`: Pearson correlation over the positions where - /// both sides are numbers. - fn evaluate_correl(&self, arguments: &[Expr]) -> Value { + /// Correlation or covariance over positions where both sides are numbers. + fn evaluate_paired_statistics(&self, function: Function, arguments: &[Expr]) -> Value { if arguments.len() != 2 { return Value::Error(CalcError::InvalidArguments); } @@ -2232,7 +2286,12 @@ impl Workbook { _ => None, }) .collect(); - match correlation(&pairs) { + let result = if function == Function::Correl { + correlation(&pairs) + } else { + covariance(&pairs, function == Function::CovarianceS) + }; + match result { Ok(value) => Value::Number(value), Err(error) => Value::Error(error), } @@ -2912,7 +2971,10 @@ fn is_elementwise(function: Function) -> bool { | Function::YearFrac | Function::Days360 | Function::Pmt + | Function::Pv | Function::NormDist + | Function::NormSDist + | Function::NormSDistLegacy | Function::IsBlank | Function::IsNumber | Function::IsText @@ -3287,6 +3349,94 @@ fn payment( Ok(-(present * growth + future) * rate / (timing * (growth - 1.0))) } +fn present_value( + rate: f64, + periods: f64, + payment: f64, + future: f64, + at_start: bool, +) -> Result { + if ![rate, periods, payment, future] + .iter() + .all(|value| value.is_finite()) + { + return Err(CalcError::InvalidNumber); + } + let result = if rate == 0.0 { + -payment * periods - future + } else { + let growth = (1.0 + rate).powf(periods); + if growth == 0.0 { + return Err(CalcError::DivisionByZero); + } + let growth_minus_one = if rate > -1.0 { + (periods * rate.ln_1p()).exp_m1() + } else { + growth - 1.0 + }; + -(future + payment * if at_start { 1.0 + rate } else { 1.0 } * growth_minus_one / rate) + / growth + }; + if result.is_finite() { + Ok(result) + } else { + Err(CalcError::InvalidNumber) + } +} + +/// Excel's periodic IRR iteration, keeping text/blanks out of the period count. +fn periodic_internal_rate_of_return(values: &[f64], guess: f64) -> Result { + if !guess.is_finite() + || guess <= -1.0 + || !values.iter().any(|value| *value > 0.0) + || !values.iter().any(|value| *value < 0.0) + { + return Err(CalcError::InvalidNumber); + } + let mut rate = guess; + for _ in 0..20 { + let base = 1.0 + rate; + let mut value = 0.0; + let mut slope = 0.0; + for (period, cash) in values.iter().enumerate() { + let discount = base.powi(period as i32); + value += cash / discount; + slope -= period as f64 * cash / (discount * base); + } + if slope == 0.0 || !slope.is_finite() { + return Err(CalcError::InvalidNumber); + } + let next = rate - value / slope; + if !next.is_finite() || next <= -1.0 { + return Err(CalcError::InvalidNumber); + } + if (next - rate).abs() < 1.0e-9 { + return Ok(next); + } + rate = next; + } + Err(CalcError::InvalidNumber) +} + +fn covariance(pairs: &[(f64, f64)], sample: bool) -> Result { + let correction = usize::from(sample); + if pairs.len() <= correction { + return Err(CalcError::DivisionByZero); + } + let mean_x = pairs.iter().map(|(x, _)| x).sum::() / pairs.len() as f64; + let mean_y = pairs.iter().map(|(_, y)| y).sum::() / pairs.len() as f64; + let value = pairs + .iter() + .map(|(x, y)| (x - mean_x) * (y - mean_y)) + .sum::() + / (pairs.len() - correction) as f64; + if value.is_finite() { + Ok(value) + } else { + Err(CalcError::InvalidNumber) + } +} + /// `NPV`: cash flows discounted from the end of the first period. fn net_present_value(rate: f64, values: &[f64]) -> Result { if rate == -1.0 { @@ -4111,6 +4261,22 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { fn parse_postfix(&mut self) -> Result { let mut expression = self.parse_primary()?; + loop { + self.skip_space(); + if self.peek() != Some(b':') { + break; + } + self.offset += 1; + // In Sheet!A1:B2 the unqualified second endpoint inherits Sheet. + let original_sheet = self.sheet; + self.sheet = match &expression { + Expr::Reference(cell) | Expr::Range { anchor: cell, .. } => cell.sheet, + _ => self.sheet, + }; + let right = self.parse_primary(); + self.sheet = original_sheet; + expression = join_reference_range(expression, right?)?; + } loop { self.skip_space(); if self.peek() != Some(b'%') { @@ -4316,7 +4482,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { return Ok(Expr::Boolean(false)); } match parse_a1(token, self.sheet) { - Ok(first) => self.parse_range_tail(first, self.sheet), + Ok(first) => Ok(Expr::Reference(first)), Err(_) => self.parse_defined_name(token), } } @@ -4439,7 +4605,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { } let token = &self.source[start..self.offset]; match parse_a1(token, sheet) { - Ok(first) => self.parse_range_tail(first, sheet), + Ok(first) => Ok(Expr::Reference(first)), Err(_) if self .defined_names @@ -4452,21 +4618,6 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { } } - fn parse_range_tail(&mut self, first: CellId, sheet: u32) -> Result { - self.skip_space(); - if self.peek() != Some(b':') { - return Ok(Expr::Reference(first)); - } - self.offset += 1; - self.skip_space(); - let second_start = self.offset; - while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'$') { - self.offset += 1; - } - let second = parse_a1(&self.source[second_start..self.offset], sheet)?; - expand_range(first, second) - } - fn resolve_sheet(&self, name: &str) -> Result { self.sheet_names .get(&name.to_lowercase()) @@ -4597,10 +4748,18 @@ const FUNCTION_REGISTRY: &[(&str, Function)] = &[ ("WORKDAY", Function::WorkDay), ("LOOKUP", Function::Lookup), ("PMT", Function::Pmt), + ("PV", Function::Pv), + ("IRR", Function::Irr), ("NPV", Function::Npv), ("XNPV", Function::Xnpv), ("XIRR", Function::Xirr), ("NORMDIST", Function::NormDist), + ("NORM.DIST", Function::NormDist), + ("NORMSDIST", Function::NormSDistLegacy), + ("NORM.S.DIST", Function::NormSDist), + ("COVAR", Function::CovarianceP), + ("COVARIANCE.P", Function::CovarianceP), + ("COVARIANCE.S", Function::CovarianceS), ("AVERAGEA", Function::AverageA), ("CORREL", Function::Correl), ("ISBLANK", Function::IsBlank), @@ -4804,10 +4963,171 @@ fn expand_range(first: CellId, second: CellId) -> Result { }) } +fn join_reference_range(left: Expr, right: Expr) -> Result { + fn bounds(expression: &Expr) -> Option<(CellId, CellId)> { + match expression { + Expr::Reference(cell) => Some((*cell, *cell)), + Expr::Range { + anchor, + rows, + columns, + members: None, + } => Some(( + *anchor, + CellId::new( + anchor.sheet, + anchor.row + *rows as u32 - 1, + anchor.column + *columns as u32 - 1, + ), + )), + _ => None, + } + } + let left_bounds = bounds(&left); + let right_bounds = bounds(&right); + let left_deleted = matches!(left, Expr::Error(CalcError::InvalidReference)); + let right_deleted = matches!(right, Expr::Error(CalcError::InvalidReference)); + if (left_deleted && (right_deleted || right_bounds.is_some())) + || (right_deleted && left_bounds.is_some()) + { + return Ok(Expr::Error(CalcError::InvalidReference)); + } + let (Some((first, first_end)), Some((second, second_end))) = (left_bounds, right_bounds) else { + return Err(FormulaError::InvalidReference( + "range endpoints must be references".into(), + )); + }; + if first.sheet != second.sheet { + return Err(FormulaError::InvalidReference( + "range endpoints cross sheets".into(), + )); + } + expand_range( + CellId::new( + first.sheet, + first.row.min(second.row), + first.column.min(second.column), + ), + CellId::new( + first.sheet, + first_end.row.max(second_end.row), + first_end.column.max(second_end.column), + ), + ) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn deleted_and_qualified_range_endpoints_follow_reference_semantics() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Input"); + workbook.define_sheet(1, "Other sheet"); + workbook.set_number(CellId::new(1, 0, 0), 3.0); + workbook.set_number(CellId::new(1, 1, 0), 7.0); + for formula in [ + "=SUM('Other sheet'!A1:A2)", + "=SUM('Other sheet'!A1:'Other sheet'!A2)", + "=SUM(('Other sheet'!A1):('Other sheet'!A2))", + ] { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(10.0), "{formula}"); + } + let changed = workbook.set_number(CellId::new(1, 1, 0), 9.0); + assert!(changed.evaluated.contains(&cell(0, 3))); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(12.0)); + for formula in [ + "=SUM(#REF!:#REF!)", + "=SUM(A1:#REF!)", + "=SUM(#REF!:A1)", + "=SUM('Other sheet'!#REF!:A2)", + "=SUM('Other sheet'!A1:'Other sheet'!#REF!)", + ] { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!( + workbook.value(cell(0, 3)), + Value::Error(CalcError::InvalidReference), + "{formula}" + ); + } + workbook + .set_formula(cell(0, 3), "=IFERROR(SUM(#REF!:#REF!),17)") + .unwrap(); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(17.0)); + for formula in [ + "=SUM(Input!A1:'Other sheet'!A2)", + "=SUM(A1:2)", + "=SUM(1:A2)", + "=SUM(A1:[1]Sheet!B2)", + ] { + assert!( + workbook.set_formula(cell(0, 3), formula).is_err(), + "{formula}" + ); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(17.0)); + } + } + + #[test] + fn added_financial_and_statistical_functions_match_reference_values() { + let mut workbook = Workbook::default(); + let examples = [ + ("=PV(0,10,-100)", 1000.0, 1e-10), + ("=PV(0,0,0,100)", -100.0, 1e-10), + ("=PV(0.1,1,-110)", 100.0, 1e-10), + ("=PV(0.1,1,-100,,1)", 100.0, 1e-10), + ("=PV(0.08/12,240,500,,0)", -59777.15, 0.005), + ("=PV(1e-12,1,-100)", 100.0 / (1.0 + 1e-12), 1e-10), + ("=IRR({-100,110})", 0.1, 1e-9), + ("=IRR({-100,\"ignore\",FALSE,0,121})", 0.1, 1e-9), + ("=IRR({-100,90})", -0.1, 1e-9), + ( + "=IRR({-70000,12000,15000,18000,21000,26000})", + 0.086630948036531, + 1e-9, + ), + ("=COVAR({3,2,4,5,6},{9,7,12,15,17})", 5.2, 1e-10), + ("=COVARIANCE.P({0,2},{2,0})", -1.0, 1e-10), + ("=COVARIANCE.S({0,2},{2,0})", -2.0, 1e-10), + ("=COVAR({1,\"ignored\",3},{2,999,6})", 2.0, 1e-10), + ("=NORMSDIST(0)", 0.5, 1e-10), + ("=_xlfn.NORM.S.DIST(1.333333,TRUE)", 0.908788726, 1e-9), + ("=NORM.S.DIST(1.333333,FALSE)", 0.164010148, 1e-9), + ( + "=NORM.DIST(0,0,1,FALSE)", + 1.0 / (2.0 * std::f64::consts::PI).sqrt(), + 1e-10, + ), + ]; + for (formula, expected, tolerance) in examples { + workbook.set_formula(cell(0, 0), formula).unwrap(); + let Value::Number(value) = workbook.value(cell(0, 0)) else { + panic!("{formula} did not return a number") + }; + assert!( + (value - expected).abs() < tolerance, + "{formula}: {value} != {expected}" + ); + } + for (formula, error) in [ + ("=IRR({1,2})", CalcError::InvalidNumber), + ("=IRR({-1,1},-1)", CalcError::InvalidNumber), + ("=IRR({-1,#N/A,2})", CalcError::NotAvailable), + ("=PV(-1,1,1)", CalcError::DivisionByZero), + ("=PV(0,1,#REF!)", CalcError::InvalidReference), + ("=COVARIANCE.S({1},{2})", CalcError::DivisionByZero), + ("=COVAR({1,2},{3})", CalcError::NotAvailable), + ("=COVAR({1,#N/A},{3,4})", CalcError::NotAvailable), + ("=NORMSDIST(0,FALSE)", CalcError::InvalidArguments), + ("=NORM.S.DIST(0)", CalcError::InvalidArguments), + ] { + workbook.set_formula(cell(0, 0), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 0)), Value::Error(error), "{formula}"); + } + } + #[test] fn array_constants_feed_aggregates_broadcasting_and_lookups() { let mut workbook = Workbook::default(); diff --git a/crates/omasheets-service/tests/native_controls.rs b/crates/omasheets-service/tests/native_controls.rs index f903033..ae2078a 100644 --- a/crates/omasheets-service/tests/native_controls.rs +++ b/crates/omasheets-service/tests/native_controls.rs @@ -89,6 +89,36 @@ fn range(row: usize, column: usize, rows: usize, columns: usize) -> Value { json!({"row":row,"column":column,"rows":rows,"columns":columns}) } +#[test] +fn array_and_financial_formulas_keep_stable_bindings_through_sort_and_reopen() { + let mut f = Fixture::new(); + f.number("A1", 10.0); + f.number("A2", 20.0); + f.formula("B2", "=SUM(A2*{2,3})+PV(0,1,-5)"); + f.formula("E5", "=IFERROR(SUM(#REF!:#REF!),7)"); + assert_eq!(f.cell("B2")["value"]["value"], 105.0); + f.edit( + json!({"action":"sort","range":range(0,0,2,1),"column":0,"header":false,"descending":true}), + ); + let page = f.page(); + let moved = page["cells"] + .as_array() + .unwrap() + .iter() + .find(|cell| cell["a1"] == "B1") + .unwrap(); + assert_eq!(moved["formula"], "=SUM(A1*{2,3})+PV(0,1,-5)"); + assert_eq!(moved["value"]["value"], 105.0); + assert!(moved.get("formula_projection_error").is_none()); + let revision = f.call(json!({"kind":"revision"})); + f.reopen(); + assert_eq!(f.call(json!({"kind":"revision"})), revision); + assert_eq!(f.cell("B1")["value"]["value"], 105.0); + assert_eq!(f.cell("E5")["value"]["value"], 7.0); + f.number("A1", 30.0); + assert_eq!(f.cell("B1")["value"]["value"], 155.0); +} + #[test] fn moved_formula_text_preserves_absolute_axes_and_refuses_nonrectangular_ranges() { let mut f = Fixture::new(); diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index 659894c..b45dfe7 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -1119,6 +1119,43 @@ mod tests { assert_eq!(formula.formula.as_deref(), Some("A1+A2")); } + #[test] + fn syntax_diagnostics_expose_only_fixed_token_classes() { + let path = temporary_xlsx(&package( + &[], + "", + r#"SUM(1;2)3SUM({1,})1"#, + "", + )); + let imported = import_xlsx(&path, ImportLimits::default()).unwrap(); + std::fs::remove_file(path).unwrap(); + let report = imported.report(); + assert_eq!( + report.syntax_failure_tokens, + BTreeMap::from([("semicolon".into(), 1), ("array_brace".into(), 1),]) + ); + assert_eq!(report.unsupported_reasons["syntax"], 2); + assert_eq!(report.formula_cells_compared, 1); + } + + #[test] + fn array_financial_and_deleted_reference_formulas_match_xlsx_caches() { + let path = temporary_xlsx(&package( + &[], + r#"{2;3}0.1"#, + r#"SUMPRODUCT({10;20},weights)80PV(Data!rate,1,-110)100_xlfn.XLOOKUP(2,{1,2},{10,20})20SUM(#REF!:#REF!)#REF!IFERROR(SUM(A1:#REF!),17)17"#, + "", + )); + let imported = import_xlsx(&path, ImportLimits::default()).unwrap(); + std::fs::remove_file(path).unwrap(); + let report = imported.report(); + assert_eq!(report.formula_cells_observed, 6); + assert_eq!(report.formula_cells_loaded, 6); + assert_eq!(report.stored_values_matched, 6); + assert_eq!(report.stored_values_mismatched, 0); + assert!(report.unsupported_reasons.is_empty()); + } + #[test] fn shared_formulas_expand_from_their_anchor_cell() { // The shared group is anchored at B5 with ref A5:B6; A5 carries its diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 4167db0..403792a 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -1,14 +1,16 @@ # Supported formula functions The owned M0 engine (`crates/omasheets-calc`) accepts exactly the -96 function names listed below, grouped for reading. +104 function names listed below, grouped for reading. A test in the calc crate fails when this file and the registry disagree, so the count here is never edited by hand: add the function to the registry and regenerate this list. Operators: `+ - * / ^ & %`, unary `+`/`-`, comparisons `= <> < <= > >=`, error literals (`#REF!`, `#N/A`, `#DIV/0!`, `#VALUE!`, `#NUM!`, `#NAME?`, -`#NULL!`, and `Sheet!#REF!` for a deleted cell on another sheet), omitted arguments, bounded rectangular ranges, absolute markers, +`#NULL!`, and `Sheet!#REF!` for a deleted cell on another sheet), omitted arguments, +bounded rectangular ranges (including qualified endpoints and deleted endpoints +such as `A1:#REF!`, which evaluate to `#REF!`), absolute markers, cross-sheet references, workbook and sheet-scoped defined names (including `Sheet!LocalName`; tokens past the grid such as `Table1` are names), implicit intersection of a range in @@ -56,6 +58,12 @@ unsorted keys are undefined in Excel and are not promised here. - `AVERAGEA` - `CORREL` - `NORMDIST` +- `NORM.DIST` +- `NORMSDIST` +- `NORM.S.DIST` +- `COVAR` +- `COVARIANCE.P` +- `COVARIANCE.S` ### Conditional aggregates @@ -149,6 +157,8 @@ unsorted keys are undefined in Excel and are not promised here. ### Financial - `PMT` +- `PV` +- `IRR` - `NPV` - `XNPV` - `XIRR` diff --git a/docs/NATIVE-SPREADSHEET-SUPPORT.md b/docs/NATIVE-SPREADSHEET-SUPPORT.md index 48802f0..0f69a87 100644 --- a/docs/NATIVE-SPREADSHEET-SUPPORT.md +++ b/docs/NATIVE-SPREADSHEET-SUPPORT.md @@ -56,7 +56,9 @@ The width calculation follows the [SpreadsheetML column specification](https://l ## Formula and editing boundaries -The parser registry contains 96 function names, including TEXTJOIN. The +The parser registry contains 104 function names, including TEXTJOIN, PV, IRR, +covariance and standard-normal distribution functions. Bounded array constants +work in aggregates and lookups without spilling into neighbouring cells. The registry and [function list](FUNCTIONS.md) are checked together. Clock and random functions remain explicitly refused: calculation does not read hidden time or randomness. The core's stored tick events have not been connected to volatile From a656da6161a624ee5acba025264f9c861e343530 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 18:16:35 +0100 Subject: [PATCH 4/9] Record measured frozen-corpus coverage increase --- corpus/sources/README.md | 46 +++++- .../sources/enron-figshare.score-delta.json | 152 ++++++++++-------- .../sources/enron-figshare.score-summary.json | 133 +++++++-------- docs/V0.1-RELEASE.md | 19 ++- 4 files changed, 195 insertions(+), 155 deletions(-) diff --git a/corpus/sources/README.md b/corpus/sources/README.md index d9ca704..20a17ca 100644 --- a/corpus/sources/README.md +++ b/corpus/sources/README.md @@ -28,8 +28,44 @@ the frozen sample for the current engine (engine commit, wall time, process-tree peak memory, both lane summaries, the owned lane's unsupported-function distribution and the failure kinds), and `enron-figshare.score-delta.json` records the owned lane against the -baseline engine that first scored the sample. Both are aggregate only. Scored -on 2026-09-01 on a 4-vCPU Linux container: +baseline engine named in that delta. Both are aggregate only; the JSON records +the exact engine revisions and runners. + +The latest comparison was scored on 2026-09-08, sequentially on one GitHub +`ubuntu-latest` runner with separate build directories. Both revisions used +the same frozen 1,000-workbook manifest and exposed 924,235 formula cells. +[Workflow evidence](https://github.com/tcballard/OmaSheets/actions/runs/34254502652). + +| Owned engine lane | Baseline (`ecf0036`) | Formula coverage (`12d1c8c`) | +|---|---:|---:| +| Workbooks opened | 996 / 1,000 | 996 / 1,000 | +| Formula cells observed | 924,235 | 924,235 | +| Loaded and compared | 825,016 (89.26%) | 829,529 (89.75%) | +| Stored values matched | 823,932 | 828,437 | +| Match rate of compared | 99.87% | 99.87% | +| Stored values mismatched | 1,084 | 1,092 | +| Not compiled | 99,219 | 94,706 | + +This adds 4,513 compiled formulas and 4,505 stored-value matches without +changing the sample, comparison tolerance or denominator. The eight additional +mismatches remain visible. The aggregate cannot attribute them to individual +formulas or establish that every previously matched cell is unchanged. + +Parser work also reveals later failures: the 10,791 syntax classifications +are gone, while invalid references rise from 4,260 to 10,551 and unsupported +function classifications rise from 26,617 to 28,967. `OFFSET` now accounts for +3,619 first failures; previously some of these stopped at a syntax or name +error. A lower count in an early failure class does not mean every affected +formula now compiles. + +The remaining first-failure groups are 47,007 external workbook references, +28,967 unsupported functions, 10,551 invalid references, 5,767 unsupported +name definitions, 2,292 cycles and 122 unknown names. Within unsupported +functions, 21,737 cells call proprietary add-ins. Ordinary gaps include +`OFFSET`, locale-sensitive `DATEVALUE`, volatile functions and `INDIRECT`. +External/add-in execution and hidden clock or random state remain refused. +These are compatibility measurements, not target-desktop performance claims. + Generate both files from a measured schema-2 scorer report with `scripts/update_corpus_summary.py`. The command validates the manifest digest, @@ -49,6 +85,12 @@ python scripts/update_corpus_summary.py \ --runner "linux x86_64, eight-core baseline, cold run" ``` +### Historical comparison + +The following 2026-09-01 run predates importer fixes that exposed additional +formula cells. Its smaller denominator makes the raw rate unsuitable for a +direct comparison with the current table. + | Owned M0 engine lane | Baseline (`08f38d1`) | After formula gaps (`0eccaff`) | |---|---|---| | Workbooks opened | 971 / 1000 | 971 / 1000 | diff --git a/corpus/sources/enron-figshare.score-delta.json b/corpus/sources/enron-figshare.score-delta.json index 598b028..85eb996 100644 --- a/corpus/sources/enron-figshare.score-delta.json +++ b/corpus/sources/enron-figshare.score-delta.json @@ -1,86 +1,108 @@ { - "schema": 1, - "source": "enron-figshare", + "baseline_engine_commit": "ecf00367004ccbeb56a736fd0aef30dd4e45c1ba", + "engine_commit": "12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5", "manifest": "enron-figshare.jsonl", "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", - "baseline_engine_commit": "3d32520f91d51b63d327202cca82920125e7d36d", - "engine_commit": "71f21c5e285c7ec98543b498088c17ee648ef858", - "scored": "2026-09-02", + "notes": [ + "Frozen 1,000-workbook manifest and 924,235-formula denominator unchanged.", + "Workflow evidence: https://github.com/tcballard/OmaSheets/actions/runs/34254502652", + "First-failure categories move as parsing advances: syntax 10,791 -> 0, invalid references 4,260 -> 10,551, unsupported functions 26,617 -> 28,967. This is not a claim that every former syntax failure now evaluates.", + "4,513 more formulas compiled; 4,505 more stored-value matches; total mismatches 1,084 -> 1,092. Aggregate evidence cannot assign the eight additional mismatches to individual formulas.", + "No performance comparison to Omarchy desktop hardware is implied." + ], "owned_lane": { - "opened": { - "before": 996, - "after": 996 + "comparison_coverage": { + "after": 0.8975303899982148, + "before": 0.8926474327416728 }, "failed": { - "before": 4, - "after": 4 - }, - "timed_out": { - "before": 0, - "after": 0 + "after": 4, + "before": 4 }, - "formula_cells_observed": { - "before": 924235, - "after": 924235 + "formula_cells_compared": { + "after": 829529, + "before": 825016 }, "formula_cells_loaded": { - "before": 817225, - "after": 825016 + "after": 829529, + "before": 825016 }, - "formula_cells_compared": { - "before": 817225, - "after": 825016 + "formula_cells_observed": { + "after": 924235, + "before": 924235 }, - "stored_values_matched": { - "before": 816141, - "after": 823932 + "formula_parse_rate": { + "after": 0.8975303899982148, + "before": 0.8926474327416728 }, - "stored_values_mismatched": { - "before": 1084, - "after": 1084 + "opened": { + "after": 996, + "before": 996 }, - "unsupported_formulas": { - "before": 107010, - "after": 99219 + "peak_rss_bytes_max": { + "after": 142831616, + "before": 142925824 }, - "formula_parse_rate": { - "before": 0.8842177584705189, - "after": 0.8926474327416728 + "stored_value_match_rate": { + "after": 0.9986835903265588, + "before": 0.9986860860880273 }, - "comparison_coverage": { - "before": 0.8842177584705189, - "after": 0.8926474327416728 + "stored_values_matched": { + "after": 828437, + "before": 823932 }, - "stored_value_match_rate": { - "before": 0.9986735599131206, - "after": 0.9986860860880273 + "stored_values_mismatched": { + "after": 1092, + "before": 1084 }, - "peak_rss_bytes_max": { - "before": 116269056, - "after": 116408320 + "timed_out": { + "after": 0, + "before": 0 + }, + "unsupported_formulas": { + "after": 94706, + "before": 99219 } }, "owned_lane_added": { - "workbooks_with_skipped_sheets": { - "before": 23, - "after": 23 - }, "skipped_sheets": { - "before": 48, - "after": 48 + "after": 48, + "before": 48 + }, + "workbooks_with_skipped_sheets": { + "after": 23, + "before": 23 } }, + "process_tree_peak_rss_bytes": { + "after": 1493118976, + "before": 1502441472 + }, + "resolved_defects": [ + "Array constants in aggregate, elementwise and lookup inputs", + "Sheet-qualified defined names and known modern function prefixes", + "Qualified and deleted reference range endpoints" + ], + "resolved_functions": [ + "COVAR", + "IRR", + "NORMSDIST", + "PV" + ], + "resolved_open_failures": {}, + "schema": 1, + "scored": "2026-09-08", + "source": "enron-figshare", "unsupported_reasons": { - "before": { - "cycle": 2286, + "after": { + "cycle": 2292, "external_reference": 47007, - "invalid_reference": 12051, - "syntax": 10791, + "invalid_reference": 10551, "unknown_name": 122, - "unsupported_function": 26617, - "unsupported_name": 8136 + "unsupported_function": 28967, + "unsupported_name": 5767 }, - "after": { + "before": { "cycle": 2286, "external_reference": 47007, "invalid_reference": 4260, @@ -90,20 +112,8 @@ "unsupported_name": 8136 } }, - "resolved_functions": [], - "resolved_defects": [ - "a sheet-qualified #REF! (a deleted cell on another sheet) was refused as an invalid reference instead of compiling to the #REF! error" - ], - "resolved_open_failures": {}, "wall_seconds": { - "before": 139.0, - "after": 208.0 - }, - "process_tree_peak_rss_bytes": { - "before": 1539121152, - "after": 1544372224 - }, - "notes": [ - "Timing evidence in this entry comes from a restarted container that is about 35% slower than the one the previous entries ran on: the candidate lane, which this engine change does not touch, went from 44 s to 60 s of summed import time. Per-cell results are identical across two runs on the new container; compare cell counts across entries, not wall seconds." - ] + "after": 117.606367, + "before": 118.763427 + } } diff --git a/corpus/sources/enron-figshare.score-summary.json b/corpus/sources/enron-figshare.score-summary.json index e23fc80..9694c01 100644 --- a/corpus/sources/enron-figshare.score-summary.json +++ b/corpus/sources/enron-figshare.score-summary.json @@ -1,53 +1,64 @@ { - "schema": 1, - "source": "enron-figshare", - "manifest": "enron-figshare.jsonl", - "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", - "scored": "2026-09-02", - "command": "omasheets-corpus score corpus/sources/enron-figshare.jsonl --timeout-seconds 30", - "engine_commit": "71f21c5e285c7ec98543b498088c17ee648ef858", - "runner": "linux x86_64 container, 4 vCPU", - "wall_seconds": 208.0, - "process_tree_peak_rss_bytes": 1544372224, "candidate_engine": "formualizer-calamine-0.8.4", + "candidate_failure_kinds": { + "chartsheet_instead_of_worksheet": 39, + "memory_allocation_failed": 2, + "other": 2, + "relationship_not_found": 24, + "undefined_name_or_formula_parse": 99 + }, "candidate_summary": { + "failed": 166, "files": 1000, + "formula_cells_loaded": 419854, + "formula_cells_observed": 419854, + "formula_parse_rate": 1, + "peak_rss_bytes_max": 154779648, "succeeded": 834, - "failed": 166, "timed_out": 0, - "value_cells_observed": 3843371, - "formula_cells_observed": 419854, - "formula_cells_loaded": 419854, - "formula_parse_rate": 1.0, - "peak_rss_bytes_max": 160321536 + "value_cells_observed": 3843371 }, + "command": "omasheets-corpus score enron-figshare.jsonl --timeout-seconds 30", + "engine_commit": "12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5", + "manifest": "enron-figshare.jsonl", + "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", + "notes": [ + "Aggregate only: no workbook paths, cell contents or per-file results are recorded here.", + "Baseline and candidate scored sequentially in isolated build directories on the same GitHub runner; unchanged frozen 1,000-workbook manifest.", + "Workflow evidence: https://github.com/tcballard/OmaSheets/actions/runs/34254502652", + "Raw coverage retains external references, proprietary add-ins, cycles and unsupported formulas in the denominator." + ], "owned_engine": "omasheets-owned-m0", + "owned_failure_kinds": { + "cell_limit_2000000_exceeded": 2, + "date_system_1904_rejected": 2 + }, "owned_summary": { + "comparison_coverage": 0.8975303899982148, + "failed": 4, "files": 1000, + "formula_cells_compared": 829529, + "formula_cells_loaded": 829529, + "formula_cells_observed": 924235, + "formula_parse_rate": 0.8975303899982148, "opened": 996, - "failed": 4, + "peak_rss_bytes_max": 142831616, + "skipped_sheets": 48, + "stored_value_match_rate": 0.9986835903265588, + "stored_values_matched": 828437, + "stored_values_mismatched": 1092, + "syntax_failure_tokens": {}, "timed_out": 0, - "formula_cells_observed": 924235, - "formula_cells_loaded": 825016, - "formula_cells_compared": 825016, - "stored_values_matched": 823932, - "stored_values_mismatched": 1084, - "unsupported_formulas": 99219, - "formula_parse_rate": 0.8926474327416728, - "comparison_coverage": 0.8926474327416728, - "stored_value_match_rate": 0.9986860860880273, + "unsupported_formulas": 94706, "unsupported_reasons": { - "cycle": 2286, + "cycle": 2292, "external_reference": 47007, - "invalid_reference": 4260, - "syntax": 10791, + "invalid_reference": 10551, "unknown_name": 122, - "unsupported_function": 26617, - "unsupported_name": 8136 + "unsupported_function": 28967, + "unsupported_name": 5767 }, - "workbooks_with_skipped_sheets": 23, - "skipped_sheets": 48, - "peak_rss_bytes_max": 116408320 + "workbooks_with_skipped_sheets": 23 }, "owned_unsupported_functions": { "_XLL.HPVAL": { @@ -62,6 +73,10 @@ "formula_cells": 3642, "workbooks": 1 }, + "OFFSET": { + "formula_cells": 3619, + "workbooks": 10 + }, "_XLL.EURO": { "formula_cells": 2992, "workbooks": 2 @@ -74,10 +89,6 @@ "formula_cells": 1412, "workbooks": 2 }, - "OFFSET": { - "formula_cells": 1250, - "workbooks": 10 - }, "CALCSKEW": { "formula_cells": 1177, "workbooks": 1 @@ -162,26 +173,10 @@ "formula_cells": 10, "workbooks": 1 }, - "IRR": { - "formula_cells": 6, - "workbooks": 1 - }, - "PV": { - "formula_cells": 6, - "workbooks": 3 - }, "MMULT": { "formula_cells": 4, "workbooks": 1 }, - "NORMSDIST": { - "formula_cells": 4, - "workbooks": 1 - }, - "COVAR": { - "formula_cells": 3, - "workbooks": 1 - }, "_XLL.XASTRIP": { "formula_cells": 2, "workbooks": 1 @@ -195,28 +190,10 @@ "workbooks": 1 } }, - "candidate_failure_kinds": { - "undefined_name_or_formula_parse": 99, - "chartsheet_instead_of_worksheet": 40, - "relationship_not_found": 24, - "memory_allocation_failed": 2, - "other": 1 - }, - "owned_failure_kinds": { - "date_system_1904_rejected": 2, - "cell_limit_2000000_exceeded": 2 - }, - "notes": [ - "Aggregate only: no workbook paths, cell contents or per-file results are recorded here.", - "The owned importer opens 23 workbooks whose xl/workbook.xml lists veryHidden sheet entries with no worksheet part (converted legacy macro modules) by skipping those entries in an in-memory copy; the 48 skipped names are reported per workbook, never silently.", - "The four remaining owned-lane failures are two 1904-date-system workbooks, refused by decision (ADR-0004), and two workbooks over the 2,000,000-cell import bound.", - "external_reference counts formulas that reach into another workbook; the engine never evaluates them.", - "Ranges are shared graph nodes with membership decided by position, and imports load in bulk with one recalculation; the worst-workbook peak resident set fell from about 1 GB to about 110 MB and the two former 30 s timeouts import in well under 10 s.", - "Twelve functions the corpus used most without support (YEARFRAC, DAYS360, NETWORKDAYS, WORKDAY, LOOKUP, PMT, NPV, XNPV, XIRR, NORMDIST, AVERAGEA, CORREL) now evaluate; two workbooks whose defined names are literally #REF! now report #REF! where Excel had kept a stale cached value.", - "Mismatch round 1: tokens past the grid resolve as names, sheet-scoped names keep their scope, criteria follow Excel for spacing, wildcards, blanks, errors and mixed types, blank lookups never match blank cells, negative zero folds to zero, INDEX passes a failed MATCH through, lookups skip error cells, math domain errors are #NUM!, and shared formulas expand from their anchor cell; the eight worst-mismatch workbooks of the previous score now match Excel cell for cell apart from three macro-backed cells.", - "Mismatch round 2: numbers compare at 15 significant digits as in Excel, and range expressions inside aggregate arguments (SUM(IF(...)), SUM(A:A*B:B), SUMPRODUCT((...)*...)) evaluate elementwise with Excel's broadcasting.", - "Sheet-qualified #REF! tokens compile to the #REF! error value, which is what Excel stores for a deleted cell on another sheet.", - "Timing evidence in this entry comes from a restarted container that is about 35% slower than the one the previous entries ran on: the candidate lane, which this engine change does not touch, went from 44 s to 60 s of summed import time. Per-cell results are identical across two runs on the new container; compare cell counts across entries, not wall seconds.", - "See enron-figshare.score-delta.json for the change against the previous engine." - ] + "process_tree_peak_rss_bytes": 1493118976, + "runner": "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware", + "schema": 1, + "scored": "2026-09-08", + "source": "enron-figshare", + "wall_seconds": 117.606367 } diff --git a/docs/V0.1-RELEASE.md b/docs/V0.1-RELEASE.md index 2b703ba..5501216 100644 --- a/docs/V0.1-RELEASE.md +++ b/docs/V0.1-RELEASE.md @@ -29,16 +29,27 @@ projections. Agents cannot append to `main`, approve a merge or publish output. | Native import | Bounded XLSX conversion stages one event transaction, publishes without replacement and verifies replay digest | Installed API proof plus dogfood workbook review confirms the manifest is sufficient | | Omarchy integration | Reproducible bundle includes the native grid, compatibility window, user-facing commands, `.omasheets` MIME routing and an explicit, reversible `setup --omarchy --enable-service` path; compiler-free acceptance exercises the grid, import, edit, branch, check, diff, merge, CSV, XLSX, Parquet, close and reopen | Installed workflow and rollback pass owner review on Omarchy | | Resource targets | Headless wiring evidence only | Declared baseline meets the M1 open, paint, scroll and idle-RSS targets | -| Corpus targets | 99.6% open, 89.26% raw parse, 99.87% stored-value match | Parse-coverage denominator is recorded and the M2 gates pass without changing the frozen sample | +| Corpus targets | 99.6% open, 89.75% raw parse, 99.87% stored-value match | Parse-coverage denominator is recorded and the M2 gates pass without changing the frozen sample | ## Required release evidence -The current frozen-sample raw parse rate is 825,016 / 924,235 (89.26%). +The frozen-sample raw parse rate is now 829,529 / 924,235 (89.75%), up +from 825,016 (89.26%) on the same sample. The 2026-09-08 sequential comparison +of `ecf0036` and `12d1c8c` compiled 4,513 additional formulas and matched +4,505 additional stored values. Total mismatches changed from 1,084 to 1,092; +stored-value agreement remains 99.87% of compared formulas. See the +[aggregate evidence](../corpus/sources/enron-figshare.score-summary.json) and +[before/after delta](../corpus/sources/enron-figshare.score-delta.json). + The 47,007 formulas classified as external references alone cap raw coverage at 94.91% while the external-workbook refusal policy remains in force, even if every other formula compiles. This is an arithmetic upper bound from the -existing aggregate summary, not a new scorer run. The roadmap's 97% target -therefore needs an explicit maintainer decision about its denominator or +aggregate summary. A further 21,737 first-failure classifications are +proprietary add-in functions, including `_XLL.HPVAL`, `INDGENCOST` and +`CALCSKEW`; implementing ordinary Excel functions cannot supply their missing +implementations. Together these classes cap raw coverage at 92.56% for this +classification of the frozen sample. This is a bound, not an adjusted score. +The roadmap's 97% target therefore needs an explicit maintainer decision about its denominator or release scope. Do not silently exclude unsupported formulas, lower the target, or describe the current corpus gate as passed. Engineering improvements to supported formulas remain useful independently of that decision. From 9f57a3d6d02305a90a771c44ea432b94c1810086 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 23:06:40 +0100 Subject: [PATCH 5/9] Classify remaining reference failures and isolate native corpus runs --- .github/workflows/corpus.yml | 8 ++-- crates/omasheets-calc/src/lib.rs | 11 +++-- crates/omasheets-corpus/src/main.rs | 22 +++++++++- crates/omasheets-xlsx/src/lib.rs | 67 +++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 8 deletions(-) diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 28e7b3d..4d6e0c1 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -56,21 +56,21 @@ jobs: mkdir -p "$CORPUS_EVIDENCE/before" "$CORPUS_EVIDENCE/after" - name: Build the baseline scorer working-directory: baseline - run: cargo build --locked --release -p omasheets-corpus --target-dir "$CORPUS_TARGETS/baseline" + run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/baseline" - name: Verify and score the baseline alone working-directory: candidate run: | "$CORPUS_TARGETS/baseline/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json" python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CORPUS_TARGETS/baseline/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30 - python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured." - name: Build the candidate scorer working-directory: candidate - run: cargo build --locked --release -p omasheets-corpus --target-dir "$CORPUS_TARGETS/candidate" + run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/candidate" - name: Score the candidate alone and report aggregate changes working-directory: candidate run: | python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CORPUS_TARGETS/candidate/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30 - python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware" + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured." python - <<'PY' import json import os diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index 4645f72..e913a1a 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -4993,9 +4993,14 @@ fn join_reference_range(left: Expr, right: Expr) -> Result { return Ok(Expr::Error(CalcError::InvalidReference)); } let (Some((first, first_end)), Some((second, second_end))) = (left_bounds, right_bounds) else { - return Err(FormulaError::InvalidReference( - "range endpoints must be references".into(), - )); + let kind = if matches!(left, Expr::Number(_)) || matches!(right, Expr::Number(_)) { + "range endpoint is number" + } else if matches!(left, Expr::Function(_, _)) || matches!(right, Expr::Function(_, _)) { + "range endpoint is function" + } else { + "range endpoints must be references" + }; + return Err(FormulaError::InvalidReference(kind.into())); }; if first.sheet != second.sheet { return Err(FormulaError::InvalidReference( diff --git a/crates/omasheets-corpus/src/main.rs b/crates/omasheets-corpus/src/main.rs index 4dadbff..1517a50 100644 --- a/crates/omasheets-corpus/src/main.rs +++ b/crates/omasheets-corpus/src/main.rs @@ -129,6 +129,8 @@ struct OwnedSummary { unsupported_functions: BTreeMap, unsupported_reasons: BTreeMap, syntax_failure_tokens: BTreeMap, + reference_failure_kinds: BTreeMap, + mismatch_value_kinds: BTreeMap, /// Workbooks the importer opened only after skipping sheet entries that /// have no worksheet part, and how many such entries it skipped in all. workbooks_with_skipped_sheets: u64, @@ -758,7 +760,11 @@ fn score_command( let owned_failed = owned_summary.failed; let report = ScoreReport { schema: 2, - engine: "formualizer-calamine-0.8.4", + engine: if cfg!(feature = "formualizer") { + "formualizer-calamine-0.8.4" + } else { + "disabled" + }, owned_engine: omasheets_xlsx::ENGINE_NAME, stored_value_comparison: "owned-m0", summary: ScoreSummary { @@ -822,6 +828,18 @@ fn accumulate_owned(summary: &mut OwnedSummary, owned: &OwnedProbe) { .entry(token.clone()) .or_default() += *cells as u64; } + for (kind, cells) in &report.reference_failure_kinds { + *summary + .reference_failure_kinds + .entry(kind.clone()) + .or_insert(0) += *cells as u64; + } + for (kind, cells) in &report.mismatch_value_kinds { + *summary + .mismatch_value_kinds + .entry(kind.clone()) + .or_insert(0) += *cells as u64; + } if !report.skipped_sheets.is_empty() { summary.workbooks_with_skipped_sheets += 1; summary.skipped_sheets += report.skipped_sheets.len() as u64; @@ -1079,6 +1097,8 @@ mod tests { 10 - loaded, )]), syntax_failure_tokens: BTreeMap::new(), + reference_failure_kinds: BTreeMap::new(), + mismatch_value_kinds: BTreeMap::new(), skipped_sheets: if loaded == 8 { vec!["Module1".to_string(), "Module2".to_string()] } else { diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index b45dfe7..dae6a34 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -114,6 +114,11 @@ pub struct ScoreReport { /// Syntax failures grouped by a fixed token class, never formula text. #[serde(default)] pub syntax_failure_tokens: BTreeMap, + /// Fixed failure classes, with no source references or cell values. + #[serde(default)] + pub reference_failure_kinds: BTreeMap, + #[serde(default)] + pub mismatch_value_kinds: BTreeMap, /// Sheet entries without a worksheet part that the importer skipped. #[serde(default)] pub skipped_sheets: Vec, @@ -217,6 +222,31 @@ impl ImportedWorkbook { pub fn report(&self) -> ScoreReport { let parity = self.parity(); + let mut reference_failure_kinds = BTreeMap::new(); + for failure in &self.unsupported { + if let FormulaError::InvalidReference(reference) = &failure.error { + let kind = match reference.as_str() { + "range endpoint is number" => "numeric_range_endpoint", + "range endpoint is function" => "dynamic_range_endpoint", + "range endpoints must be references" => "non_reference_endpoint", + "range endpoints cross sheets" => "cross_sheet_range", + "" => "empty_reference", + token if token.bytes().all(|b| b.is_ascii_alphabetic() || b == b'$') => { + "column_without_row" + } + token if token.bytes().all(|b| b.is_ascii_digit() || b == b'$') => { + "row_without_column" + } + _ => "invalid_a1", + }; + *reference_failure_kinds.entry(kind.to_string()).or_insert(0) += 1; + } + } + let mut mismatch_value_kinds = BTreeMap::new(); + for (_, stored, calculated) in self.mismatched_cells() { + let kind = format!("{} -> {}", value_kind(stored), value_kind(&calculated)); + *mismatch_value_kinds.entry(kind).or_insert(0) += 1; + } let mut syntax_failure_tokens = BTreeMap::new(); for failure in &self.unsupported { let FormulaError::UnexpectedToken(offset) = failure.error else { @@ -264,6 +294,8 @@ impl ImportedWorkbook { unsupported_functions: self.unsupported_functions(), unsupported_reasons: self.unsupported_reasons(), syntax_failure_tokens, + reference_failure_kinds, + mismatch_value_kinds, skipped_sheets: self.skipped_sheets.clone(), } } @@ -931,6 +963,16 @@ fn bounded_formula_error(error: &FormulaError) -> String { error.to_string().chars().take(256).collect() } +fn value_kind(value: &Value) -> &str { + match value { + Value::Blank => "blank", + Value::Number(_) => "number", + Value::Boolean(_) => "boolean", + Value::Text(_) => "text", + Value::Error(error) => error.label(), + } +} + fn values_match(stored: &Value, calculated: &Value) -> bool { match (stored, calculated) { (Value::Number(stored), Value::Number(calculated)) => { @@ -1119,6 +1161,31 @@ mod tests { assert_eq!(formula.formula.as_deref(), Some("A1+A2")); } + #[test] + fn reference_diagnostics_expose_only_fixed_failure_classes() { + let bytes = package( + &[], + "", + r#"SUM(1:2)0SUM(Data!A:A)0SUM(A1:INDEX(A1:A2,2))01+19"#, + "", + ); + let path = temporary_xlsx(&bytes); + let report = import_xlsx(&path).unwrap().report(); + std::fs::remove_file(path).unwrap(); + assert_eq!( + report.reference_failure_kinds, + BTreeMap::from([ + ("column_without_row".into(), 1), + ("dynamic_range_endpoint".into(), 1), + ("numeric_range_endpoint".into(), 1), + ]) + ); + assert_eq!( + report.mismatch_value_kinds, + BTreeMap::from([("number -> number".into(), 1)]) + ); + } + #[test] fn syntax_diagnostics_expose_only_fixed_token_classes() { let path = temporary_xlsx(&package( From 8b3423b567d22e1945d179434f667c702d565910 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 23:28:37 +0100 Subject: [PATCH 6/9] Support reference-valued INDEX selections and range endpoints --- crates/omasheets-calc/src/lib.rs | 180 ++++++---- crates/omasheets-calc/src/reference.rs | 335 ++++++++++++++++++ .../tests/native_controls.rs | 18 + crates/omasheets-xlsx/src/lib.rs | 24 +- docs/FUNCTIONS.md | 9 + 5 files changed, 497 insertions(+), 69 deletions(-) create mode 100644 crates/omasheets-calc/src/reference.rs diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index e913a1a..eee20a0 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -7,6 +7,7 @@ //! Dates are Excel 1900-system serial numbers; see [`serial_date`] for the //! boundary rules and the deliberately unsupported cases. +mod reference; pub mod serial_date; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; @@ -298,6 +299,8 @@ enum BinaryOp { #[derive(Clone, Copy, Debug, PartialEq)] enum Function { + /// Internal reference operator; deliberately absent from the function registry. + ReferenceSpan, Sum, Average, Min, @@ -1453,6 +1456,12 @@ impl Workbook { } fn evaluate_function(&self, function: Function, arguments: &[Expr]) -> Value { + if function == Function::ReferenceSpan { + return match self.reference_span(arguments) { + Ok(reference) => reference.scalar(self), + Err(error) => Value::Error(error), + }; + } if function == Function::If { if !matches!(arguments.len(), 2 | 3) { return Value::Error(CalcError::InvalidArguments); @@ -1796,7 +1805,8 @@ impl Workbook { | Function::IsNa | Function::HLookup | Function::Row - | Function::Column => Value::Error(CalcError::InvalidArguments), + | Function::Column + | Function::ReferenceSpan => Value::Error(CalcError::InvalidArguments), } } @@ -2120,13 +2130,15 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some(input) = ArrayInput::new(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; let (rows, columns) = input.shape(); let (candidates, results): (Vec, Vec) = if arguments.len() == 3 { - let Some(result_input) = ArrayInput::new(&arguments[2]) else { - return Value::Error(CalcError::InvalidArguments); + let result_input = match ArrayInput::new(&arguments[2], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; let (result_rows, result_columns) = result_input.shape(); if (rows != 1 && columns != 1) @@ -2346,6 +2358,16 @@ impl Workbook { } } }, + Expr::Function(Function::ReferenceSpan | Function::Index, _) => { + match self.reference_view(argument) { + Ok(reference) => reference.visit(self, &mut visit), + Err(CalcError::InvalidArguments) => match self.evaluate_array(argument) { + Ok(array) => array.values.iter().for_each(&mut visit), + Err(error) => visit(&Value::Error(error)), + }, + Err(error) => visit(&Value::Error(error)), + } + } Expr::Empty => {} other if contains_array_operand(other) => match self.evaluate_array(other) { Ok(array) => array.values.iter().for_each(&mut visit), @@ -2392,6 +2414,10 @@ impl Workbook { fn evaluate_array(&self, expression: &Expr) -> Result { match expression { Expr::Array(array) => Ok(array.clone()), + Expr::Function(Function::Index, arguments) => self.index_array(arguments), + Expr::Function(Function::ReferenceSpan, arguments) => self + .reference_span(arguments) + .map(|reference| reference.array(self)), Expr::RangeNode { node, rows, @@ -2561,35 +2587,17 @@ impl Workbook { fn evaluate_lookup_function(&self, function: Function, arguments: &[Expr]) -> Value { match function { Function::Index if matches!(arguments.len(), 2 | 3) => { - let Some(input) = ArrayInput::new(&arguments[0]) else { - return Value::Error(CalcError::InvalidArguments); - }; - let (rows, columns) = input.shape(); - // An error row (a failed MATCH) is the result, as in Excel. - let row = match positive_index(self.evaluate(&arguments[1])) { - Ok(row) => row, - Err(error) => return Value::Error(error), - }; - let column = if arguments.len() == 3 { - match positive_index(self.evaluate(&arguments[2])) { - Ok(column) => column, + if !matches!(arguments[0], Expr::Array(_)) { + match self.reference_view(&Expr::Function(function, arguments.to_vec())) { + Ok(reference) => return reference.scalar(self), + Err(CalcError::InvalidArguments) => {} Err(error) => return Value::Error(error), } - } else if columns == 1 { - 1 - } else if rows == 1 { - return if row <= columns { - input.value(self, row - 1) - } else { - Value::Error(CalcError::InvalidReference) - }; - } else { - return Value::Error(CalcError::InvalidArguments); - }; - if row > rows || column > columns { - return Value::Error(CalcError::InvalidReference); } - input.value(self, (row - 1) * columns + column - 1) + match self.index_array(arguments) { + Ok(array) => array.values[0].clone(), + Err(error) => Value::Error(error), + } } Function::Match if matches!(arguments.len(), 2 | 3) => { let mode = if arguments.len() == 3 { @@ -2604,8 +2612,9 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some(input) = ArrayInput::new(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; let (rows, columns) = input.shape(); if rows != 1 && columns != 1 { @@ -2638,8 +2647,9 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some(input) = ArrayInput::new(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; let (rows, columns) = input.shape(); let Ok(offset) = positive_index(self.evaluate(&arguments[2])) else { @@ -2672,11 +2682,13 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some(lookup_input) = ArrayInput::new(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let lookup_input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; - let Some(return_input) = ArrayInput::new(&arguments[2]) else { - return Value::Error(CalcError::InvalidArguments); + let return_input = match ArrayInput::new(&arguments[2], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; let (lookup_rows, lookup_columns) = lookup_input.shape(); let (return_rows, return_columns) = return_input.shape(); @@ -2832,22 +2844,35 @@ enum ArrayInput<'a> { columns: usize, }, Constant(&'a ArrayValue), + Selection(reference::ReferenceView), + Computed(ArrayValue), } impl<'a> ArrayInput<'a> { - fn new(expression: &'a Expr) -> Option { + fn new(expression: &'a Expr, workbook: &Workbook) -> Result { match expression { Expr::RangeNode { node, rows, columns, - } => Some(Self::Range { + } => Ok(Self::Range { node: *node, rows: *rows, columns: *columns, }), - Expr::Array(array) => Some(Self::Constant(array)), - _ => None, + Expr::Array(array) => Ok(Self::Constant(array)), + Expr::Reference(_) | Expr::Function(Function::ReferenceSpan, _) => { + workbook.reference_view(expression).map(Self::Selection) + } + Expr::Function(Function::Index, _) => match workbook.reference_view(expression) { + Ok(reference) => Ok(Self::Selection(reference)), + Err(CalcError::InvalidArguments) => { + workbook.evaluate_array(expression).map(Self::Computed) + } + Err(error) => Err(error), + }, + Expr::Error(error) => Err(error.clone()), + _ => Err(CalcError::InvalidArguments), } } @@ -2855,6 +2880,8 @@ impl<'a> ArrayInput<'a> { match self { Self::Range { rows, columns, .. } => (*rows, *columns), Self::Constant(array) => array.shape(), + Self::Selection(reference) => (reference.rows, reference.columns), + Self::Computed(array) => array.shape(), } } @@ -2862,6 +2889,8 @@ impl<'a> ArrayInput<'a> { match self { Self::Range { node, .. } => workbook.range_value(*node, index), Self::Constant(array) => array.values[index].clone(), + Self::Selection(reference) => reference.value(workbook, index), + Self::Computed(array) => array.values[index].clone(), } } @@ -2869,6 +2898,8 @@ impl<'a> ArrayInput<'a> { match self { Self::Range { node, .. } => workbook.range_values(*node), Self::Constant(array) => array.values.clone(), + Self::Selection(reference) => reference.array(workbook).values, + Self::Computed(array) => array.values.clone(), } } } @@ -2914,7 +2945,9 @@ fn broadcast_shape(shapes: &[(usize, usize)]) -> Result<(usize, usize), CalcErro /// lookups, criteria functions) produce one value and stop the search. fn contains_array_operand(expression: &Expr) -> bool { match expression { - Expr::RangeNode { .. } | Expr::Array(_) => true, + Expr::RangeNode { .. } + | Expr::Array(_) + | Expr::Function(Function::Index | Function::ReferenceSpan, _) => true, Expr::UnaryMinus(inner) | Expr::Percent(inner) => contains_array_operand(inner), Expr::Binary(_, left, right) => { contains_array_operand(left) || contains_array_operand(right) @@ -4271,7 +4304,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { let original_sheet = self.sheet; self.sheet = match &expression { Expr::Reference(cell) | Expr::Range { anchor: cell, .. } => cell.sheet, - _ => self.sheet, + _ => reference_bounds(&expression).map_or(self.sheet, |(first, _)| first.sheet), }; let right = self.parse_primary(); self.sheet = original_sheet; @@ -4963,28 +4996,33 @@ fn expand_range(first: CellId, second: CellId) -> Result { }) } -fn join_reference_range(left: Expr, right: Expr) -> Result { - fn bounds(expression: &Expr) -> Option<(CellId, CellId)> { - match expression { - Expr::Reference(cell) => Some((*cell, *cell)), - Expr::Range { - anchor, - rows, - columns, - members: None, - } => Some(( - *anchor, - CellId::new( - anchor.sheet, - anchor.row + *rows as u32 - 1, - anchor.column + *columns as u32 - 1, - ), - )), - _ => None, +fn reference_bounds(expression: &Expr) -> Option<(CellId, CellId)> { + match expression { + Expr::Reference(cell) => Some((*cell, *cell)), + Expr::Range { + anchor, + rows, + columns, + members: None, + } => Some(( + *anchor, + CellId::new( + anchor.sheet, + anchor.row + *rows as u32 - 1, + anchor.column + *columns as u32 - 1, + ), + )), + Expr::Function(Function::Index, arguments) => arguments.first().and_then(reference_bounds), + Expr::Function(Function::ReferenceSpan, arguments) => { + arguments.get(2).and_then(reference_bounds) } + _ => None, } - let left_bounds = bounds(&left); - let right_bounds = bounds(&right); +} + +fn join_reference_range(left: Expr, right: Expr) -> Result { + let left_bounds = reference_bounds(&left); + let right_bounds = reference_bounds(&right); let left_deleted = matches!(left, Expr::Error(CalcError::InvalidReference)); let right_deleted = matches!(right, Expr::Error(CalcError::InvalidReference)); if (left_deleted && (right_deleted || right_bounds.is_some())) @@ -5007,7 +5045,7 @@ fn join_reference_range(left: Expr, right: Expr) -> Result { "range endpoints cross sheets".into(), )); } - expand_range( + let envelope = expand_range( CellId::new( first.sheet, first.row.min(second.row), @@ -5018,7 +5056,15 @@ fn join_reference_range(left: Expr, right: Expr) -> Result { first_end.row.max(second_end.row), first_end.column.max(second_end.column), ), - ) + )?; + if matches!(left, Expr::Function(_, _)) || matches!(right, Expr::Function(_, _)) { + Ok(Expr::Function( + Function::ReferenceSpan, + vec![left, right, envelope], + )) + } else { + Ok(envelope) + } } #[cfg(test)] diff --git a/crates/omasheets-calc/src/reference.rs b/crates/omasheets-calc/src/reference.rs new file mode 100644 index 0000000..17d10ef --- /dev/null +++ b/crates/omasheets-calc/src/reference.rs @@ -0,0 +1,335 @@ +//! Reference-valued selections retain a bounded, stable dependency envelope. +use super::*; + +#[derive(Clone, Copy)] +pub(super) struct ReferenceView { + node: Option, + anchor: CellId, + row: usize, + column: usize, + pub(super) rows: usize, + pub(super) columns: usize, +} + +impl ReferenceView { + fn cell(self, workbook: &Workbook, row: usize, column: usize) -> CellId { + if let Some(node) = self.node { + match workbook.range_shape(node) { + RangeShape::Rectangle { anchor, .. } => CellId::new( + anchor.sheet, + anchor.row + (self.row + row) as u32, + anchor.column + (self.column + column) as u32, + ), + RangeShape::Members { columns, .. } => { + let index = (self.row + row) * columns + self.column + column; + workbook.cells[workbook.cells[node].dependencies[index]].id + } + } + } else { + CellId::new( + self.anchor.sheet, + self.anchor.row + (self.row + row) as u32, + self.anchor.column + (self.column + column) as u32, + ) + } + } + + fn position(self, workbook: &Workbook, cell: CellId) -> Option<(usize, usize)> { + let (row, column) = match self.node.map(|node| (node, workbook.range_shape(node))) { + Some((node, RangeShape::Members { columns, .. })) => { + let index = workbook.cells[node] + .dependencies + .iter() + .position(|index| workbook.cells[*index].id == cell)?; + (index / columns, index % columns) + } + _ => { + if cell.sheet != self.anchor.sheet { + return None; + } + ( + cell.row.checked_sub(self.anchor.row)? as usize, + cell.column.checked_sub(self.anchor.column)? as usize, + ) + } + }; + let row = row.checked_sub(self.row)?; + let column = column.checked_sub(self.column)?; + (row < self.rows && column < self.columns).then_some((row, column)) + } + + fn select(self, row: usize, column: usize, rows: usize, columns: usize) -> Self { + Self { + row: self.row + row, + column: self.column + column, + rows, + columns, + ..self + } + } + + pub(super) fn value(self, workbook: &Workbook, index: usize) -> Value { + workbook.value(self.cell(workbook, index / self.columns, index % self.columns)) + } + + pub(super) fn array(self, workbook: &Workbook) -> ArrayValue { + ArrayValue { + rows: self.rows, + columns: self.columns, + values: (0..self.rows * self.columns) + .map(|index| self.value(workbook, index)) + .collect(), + } + } + + pub(super) fn scalar(self, workbook: &Workbook) -> Value { + if self.rows == 1 && self.columns == 1 { + return self.value(workbook, 0); + } + let origin = workbook.evaluating.get(); + for index in 0..self.rows * self.columns { + let cell = self.cell(workbook, index / self.columns, index % self.columns); + if (self.columns == 1 || cell.column == origin.column) + && (self.rows == 1 || cell.row == origin.row) + { + return workbook.value(cell); + } + } + Value::Error(CalcError::InvalidValue) + } + + pub(super) fn visit(self, workbook: &Workbook, mut visit: impl FnMut(&Value)) { + if self + .node + .is_none_or(|node| matches!(workbook.range_shape(node), RangeShape::Rectangle { .. })) + { + workbook.for_each_rectangle_cell( + self.cell(workbook, 0, 0), + self.rows, + self.columns, + |_, index| visit(&workbook.cells[index].value), + ); + } else { + for index in 0..self.rows * self.columns { + visit(&self.value(workbook, index)); + } + } + } +} + +impl Workbook { + pub(super) fn reference_view( + &self, + expression: &Expr, + ) -> Result { + match expression { + Expr::Reference(index) => Ok(ReferenceView { + node: None, + anchor: self.cells[*index].id, + row: 0, + column: 0, + rows: 1, + columns: 1, + }), + Expr::RangeNode { + node, + rows, + columns, + } => Ok(ReferenceView { + node: Some(*node), + anchor: self.cells[*node].id, + row: 0, + column: 0, + rows: *rows, + columns: *columns, + }), + Expr::Function(Function::Index, arguments) if matches!(arguments.len(), 2 | 3) => { + let input = self.reference_view(&arguments[0])?; + let (row, column, rows, columns) = + self.index_selection(arguments, input.rows, input.columns)?; + Ok(input.select(row, column, rows, columns)) + } + Expr::Function(Function::ReferenceSpan, arguments) => self.reference_span(arguments), + Expr::Error(error) => Err(error.clone()), + _ => Err(CalcError::InvalidArguments), + } + } + + pub(super) fn reference_span( + &self, + arguments: &[Expr], + ) -> Result { + let [first, last, envelope] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let first = self.reference_view(first)?; + let last = self.reference_view(last)?; + let envelope = self.reference_view(envelope)?; + let endpoints = [ + first.cell(self, 0, 0), + first.cell(self, first.rows - 1, first.columns - 1), + last.cell(self, 0, 0), + last.cell(self, last.rows - 1, last.columns - 1), + ]; + let mut min_row = usize::MAX; + let mut min_column = usize::MAX; + let mut max_row = 0; + let mut max_column = 0; + for cell in endpoints { + let (row, column) = envelope + .position(self, cell) + .ok_or(CalcError::InvalidReference)?; + min_row = min_row.min(row); + min_column = min_column.min(column); + max_row = max_row.max(row); + max_column = max_column.max(column); + } + Ok(envelope.select( + min_row, + min_column, + max_row - min_row + 1, + max_column - min_column + 1, + )) + } + + fn index_selection( + &self, + arguments: &[Expr], + rows: usize, + columns: usize, + ) -> Result<(usize, usize, usize, usize), CalcError> { + if !matches!(arguments.len(), 2 | 3) { + return Err(CalcError::InvalidArguments); + } + let index = |expression: &Expr| -> Result { + let value = number(self.evaluate(expression))?; + if !value.is_finite() || value < 0.0 || value > usize::MAX as f64 { + return Err(CalcError::InvalidReference); + } + Ok(value.trunc() as usize) + }; + let mut row = index(&arguments[1])?; + let column = if let Some(column) = arguments.get(2) { + index(column)? + } else if rows == 1 { + let column = row; + row = 1; + column + } else if columns == 1 { + 1 + } else { + 0 + }; + if row > rows || column > columns { + return Err(CalcError::InvalidReference); + } + Ok(( + row.saturating_sub(1), + column.saturating_sub(1), + if row == 0 { rows } else { 1 }, + if column == 0 { columns } else { 1 }, + )) + } + + pub(super) fn index_array(&self, arguments: &[Expr]) -> Result { + let Some(first) = arguments.first() else { + return Err(CalcError::InvalidArguments); + }; + let input = ArrayInput::new(first, self)?; + let (input_rows, input_columns) = input.shape(); + let (row, column, rows, columns) = + self.index_selection(arguments, input_rows, input_columns)?; + let mut values = Vec::with_capacity(rows * columns); + for r in row..row + rows { + for c in column..column + columns { + values.push(input.value(self, r * input_columns + c)); + } + } + Ok(ArrayValue { + rows, + columns, + values, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cell(row: u32, column: u32) -> CellId { + CellId::new(0, row, column) + } + + #[test] + fn index_references_supply_dynamic_endpoints_and_whole_axes() { + let mut workbook = Workbook::default(); + for row in 0..3 { + workbook.set_number(cell(row, 0), (row + 1) as f64 * 10.0); + workbook.set_number(cell(row, 1), (row + 1) as f64); + } + workbook.set_number(cell(0, 3), 2.0); + for (source, expected) in [ + ("=SUM(A1:INDEX(A1:A3,D1))", 30.0), + ("=SUM(INDEX(A1:A3,2):INDEX(A1:A3,3))", 50.0), + ("=SUM(INDEX(A1:B3,0,2))", 6.0), + ("=SUM(INDEX(A1:B3,2,0))", 22.0), + ("=SUM(INDEX(A1:B3,0,0))", 66.0), + ("=SUM(INDEX(A1:B1,0))", 11.0), + ("=MATCH(20,INDEX(A1:B3,0,1),0)", 2.0), + ("=VLOOKUP(20,INDEX(A1:B3,0,0),2,FALSE)", 2.0), + ("=SUM(A1:INDEX(INDEX(A1:A3,0),D1))", 30.0), + ("=SUM(INDEX(A1:B3,0,1):INDEX(A1:B3,0,2))", 66.0), + ("=SUM(INDEX({1,2;3,4},0,2))", 6.0), + ("=SUM(INDEX({1,2;3,4},2,0))", 7.0), + ("=SUM(INDEX(A1:A3,0)*B1:B3)", 140.0), + ("=IFERROR(SUM(A1:INDEX(A1:A3,4)),17)", 17.0), + ] { + workbook.set_formula(cell(5, 6), source).unwrap(); + assert_eq!( + workbook.value(cell(5, 6)), + Value::Number(expected), + "{source}" + ); + } + workbook.set_formula(cell(1, 2), "=INDEX(A1:A3,0)").unwrap(); + assert_eq!(workbook.value(cell(1, 2)), Value::Number(20.0)); + workbook + .set_formula(cell(5, 6), "=SUM(A1:INDEX(A1:A3,D1))") + .unwrap(); + workbook.set_number(cell(0, 3), 3.0); + assert_eq!(workbook.value(cell(5, 6)), Value::Number(60.0)); + workbook.set_number(cell(2, 0), 40.0); + assert_eq!(workbook.value(cell(5, 6)), Value::Number(70.0)); + } + + #[test] + fn reference_envelope_rebinds_and_remains_sparse() { + let parsed = ParsedFormula::parse("=SUM(A1:INDEX(A1:A3,2))", 0, &HashMap::new()).unwrap(); + let mapped = parsed.map_references(|id| CellId::new(id.sheet, 2 - id.row, id.column)); + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 30.0); + workbook.set_number(cell(1, 0), 20.0); + workbook.set_number(cell(2, 0), 10.0); + workbook.set_parsed_formula(cell(5, 1), mapped).unwrap(); + assert_eq!(workbook.value(cell(5, 1)), Value::Number(30.0)); + workbook.set_number(cell(2, 0), 50.0); + assert_eq!(workbook.value(cell(5, 1)), Value::Number(70.0)); + + let mut sparse = Workbook::default(); + sparse.set_number(cell(0, 0), 2.0); + sparse.set_number(cell(499_999, 0), 3.0); + sparse + .set_formula(cell(0, 5), "=SUM(A1:INDEX(A1:A500000,500000))") + .unwrap(); + assert_eq!(sparse.value(cell(0, 5)), Value::Number(5.0)); + assert!(sparse.statistics().dependency_edges < 20); + sparse.set_number(cell(123_456, 0), 7.0); + assert_eq!(sparse.value(cell(0, 5)), Value::Number(12.0)); + assert!(matches!( + sparse.set_formula(cell(0, 0), "=SUM(A1:INDEX(A1:A3,2))"), + Err(FormulaError::Cycle(_)) + )); + assert_eq!(sparse.value(cell(0, 0)), Value::Number(2.0)); + } +} diff --git a/crates/omasheets-service/tests/native_controls.rs b/crates/omasheets-service/tests/native_controls.rs index ae2078a..826ef20 100644 --- a/crates/omasheets-service/tests/native_controls.rs +++ b/crates/omasheets-service/tests/native_controls.rs @@ -89,6 +89,24 @@ fn range(row: usize, column: usize, rows: usize, columns: usize) -> Value { json!({"row":row,"column":column,"rows":rows,"columns":columns}) } +#[test] +fn index_reference_ranges_keep_their_bound_rows_after_sort_and_reopen() { + let mut f = Fixture::new(); + f.number("A1", 30.0); + f.number("A2", 10.0); + f.number("A3", 20.0); + f.formula("B6", "=SUM(A1:INDEX(A1:A3,2))"); + assert_eq!(f.cell("B6")["value"]["value"], 40.0); + f.edit(json!({"action":"sort","range":range(0,0,3,1),"column":0,"header":false,"descending":false})); + assert_eq!(f.cell("B6")["value"]["value"], 40.0); + f.number("A3", 50.0); + assert_eq!(f.cell("B6")["value"]["value"], 60.0); + let revision = f.call(json!({"kind":"revision"})); + f.reopen(); + assert_eq!(f.call(json!({"kind":"revision"})), revision); + assert_eq!(f.cell("B6")["value"]["value"], 60.0); +} + #[test] fn array_and_financial_formulas_keep_stable_bindings_through_sort_and_reopen() { let mut f = Fixture::new(); diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index dae6a34..10e4294 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -1166,11 +1166,13 @@ mod tests { let bytes = package( &[], "", - r#"SUM(1:2)0SUM(Data!A:A)0SUM(A1:INDEX(A1:A2,2))01+19"#, + r#"SUM(1:2)0SUM(Data!A:A)0SUM(A1:SUM(A1:A2))01+19"#, "", ); let path = temporary_xlsx(&bytes); - let report = import_xlsx(&path).unwrap().report(); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); std::fs::remove_file(path).unwrap(); assert_eq!( report.reference_failure_kinds, @@ -1205,6 +1207,24 @@ mod tests { assert_eq!(report.formula_cells_compared, 1); } + #[test] + fn reference_valued_index_matches_xlsx_caches() { + let bytes = package( + &[], + "", + r#"SUM(A1:INDEX(A1:A2,2))5SUM(INDEX(A1:A2,0))5MATCH(3,INDEX(A1:A2,0),0)2"#, + "", + ); + let path = temporary_xlsx(&bytes); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); + std::fs::remove_file(path).unwrap(); + assert_eq!(report.formula_cells_loaded, 4); + assert_eq!(report.stored_values_matched, 4); + assert_eq!(report.stored_values_mismatched, 0); + } + #[test] fn array_financial_and_deleted_reference_formulas_match_xlsx_caches() { let path = temporary_xlsx(&package( diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 403792a..561d987 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -23,6 +23,15 @@ VLOOKUP/HLOOKUP/XLOOKUP. A scalar use takes the first value; spilling into neighbouring cells is not implemented. `_xlfn.` and `_xlfn._xlws.` prefixes resolve only to functions already in the registry. +`INDEX` also returns references: `SUM(A1:INDEX(A1:A100,D1))` follows the +selector in D1, and a zero row or column selects that entire axis. The range +operator binds a bounded envelope of all possible endpoint selections. Native +replay preserves that envelope's row and column identities, including after +sorting. Potential circular dependencies anywhere in the envelope are refused; +this conservative rule is the same as for INDEX's source range. A moved formula +whose current A1 spelling cannot preserve those identities reports a projection +refusal instead of exporting different references. + Deliberately unsupported: `TODAY`, `NOW`, `RAND` and every other volatile function (until the calculation context consumes stored tick events), external workbook references, 3D references, spilling array formulas, `INDIRECT`, `OFFSET`, From c172b45842b4c3cb48e37f9c47035d1264e355d6 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 23:46:54 +0100 Subject: [PATCH 7/9] Narrow constant INDEX dependencies after stable reference binding --- crates/omasheets-calc/src/lib.rs | 2 +- crates/omasheets-calc/src/reference.rs | 159 ++++++++++++++++++ .../tests/native_controls.rs | 6 +- docs/FUNCTIONS.md | 5 +- 4 files changed, 167 insertions(+), 5 deletions(-) diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index eee20a0..bc393e2 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -807,7 +807,7 @@ impl Workbook { cell: CellId, formula: ParsedFormula, ) -> Result { - let parsed = formula.expression; + let parsed = reference::narrow_reference_dependencies(formula.expression); let mut cells = BTreeSet::new(); let mut range_keys = Vec::new(); collect_dependencies(&parsed, &mut cells, &mut range_keys); diff --git a/crates/omasheets-calc/src/reference.rs b/crates/omasheets-calc/src/reference.rs index 17d10ef..a24b6a0 100644 --- a/crates/omasheets-calc/src/reference.rs +++ b/crates/omasheets-calc/src/reference.rs @@ -1,6 +1,125 @@ //! Reference-valued selections retain a bounded, stable dependency envelope. use super::*; +/// Narrow only provably constant axes, after stable bindings have been applied. +/// The persisted parsed shape remains unchanged, including legacy INDEX bindings. +pub(super) fn narrow_reference_dependencies(expression: Expr) -> Expr { + match expression { + Expr::UnaryMinus(inner) => { + Expr::UnaryMinus(Box::new(narrow_reference_dependencies(*inner))) + } + Expr::Percent(inner) => Expr::Percent(Box::new(narrow_reference_dependencies(*inner))), + Expr::Binary(op, left, right) => Expr::Binary( + op, + Box::new(narrow_reference_dependencies(*left)), + Box::new(narrow_reference_dependencies(*right)), + ), + Expr::Function(function, arguments) => { + let mut arguments: Vec<_> = arguments + .into_iter() + .map(narrow_reference_dependencies) + .collect(); + if function == Function::Index && matches!(arguments.len(), 2 | 3) { + narrow_index(&mut arguments); + } else if function == Function::ReferenceSpan + && arguments.len() == 3 + && matches!(arguments[2], Expr::Range { members: None, .. }) + && let (Some((a, b)), Some((c, d))) = ( + reference_bounds(&arguments[0]), + reference_bounds(&arguments[1]), + ) + && a.sheet == c.sheet + && let Ok(envelope) = expand_range( + CellId::new(a.sheet, a.row.min(c.row), a.column.min(c.column)), + CellId::new(a.sheet, b.row.max(d.row), b.column.max(d.column)), + ) + { + arguments[2] = envelope; + } + Expr::Function(function, arguments) + } + other => other, + } +} + +fn narrow_index(arguments: &mut Vec) { + let Expr::Range { + anchor, + members, + rows, + columns, + } = &arguments[0] + else { + return; + }; + let (anchor, members, rows, columns) = (*anchor, members.clone(), *rows, *columns); + let literal = |expression: &Expr, bound: usize| match expression { + Expr::Number(value) + if value.is_finite() && *value >= 1.0 && value.trunc() <= bound as f64 => + { + Some(value.trunc() as usize - 1) + } + _ => None, + }; + let horizontal = rows == 1 && arguments.len() == 2; + let selected_row = if horizontal { + None + } else { + literal(&arguments[1], rows) + }; + let selected_column = if horizontal { + literal(&arguments[1], columns) + } else { + arguments.get(2).and_then(|arg| literal(arg, columns)) + }; + if selected_row.is_none() && selected_column.is_none() { + return; + } + // Make the original omitted-column semantics explicit before changing shape. + if arguments.len() == 2 && !horizontal { + arguments.push(Expr::Number(if columns == 1 { 1.0 } else { 0.0 })); + } + let first_row = selected_row.unwrap_or(0); + let first_column = selected_column.unwrap_or(0); + let new_rows = if selected_row.is_some() { 1 } else { rows }; + let new_columns = if selected_column.is_some() { + 1 + } else { + columns + }; + let (new_anchor, new_members) = if let Some(members) = members { + let selected: Vec<_> = (first_row..first_row + new_rows) + .flat_map(|row| { + let members = &members; + (first_column..first_column + new_columns) + .map(move |column| members[row * columns + column]) + }) + .collect(); + (selected[0], Some(selected)) + } else { + ( + CellId::new( + anchor.sheet, + anchor.row + first_row as u32, + anchor.column + first_column as u32, + ), + None, + ) + }; + arguments[0] = Expr::Range { + anchor: new_anchor, + members: new_members, + rows: new_rows, + columns: new_columns, + }; + if selected_row.is_some() || (horizontal && selected_column.is_some()) { + arguments[1] = Expr::Number(1.0); + } + if !horizontal && selected_column.is_some() { + arguments[2] = Expr::Number(1.0); + } +} + #[derive(Clone, Copy)] pub(super) struct ReferenceView { node: Option, @@ -303,6 +422,46 @@ mod tests { assert_eq!(workbook.value(cell(5, 6)), Value::Number(70.0)); } + #[test] + fn constant_index_axes_do_not_create_false_cycles() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 10.0); + workbook.set_number(cell(1, 0), 20.0); + workbook.set_number(cell(2, 0), 30.0); + workbook.set_number(cell(0, 3), 2.0); + workbook + .set_formula(cell(0, 1), "=SUM(A1:INDEX(A1:B3,D1,1))") + .unwrap(); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(30.0)); + workbook.set_number(cell(0, 3), 3.0); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(60.0)); + workbook.set_number(cell(2, 0), 40.0); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(70.0)); + for (formula, expected) in [ + ("=SUM(INDEX(A1:B3,2))", 20.0), + ("=INDEX(A1:B1,1)", 10.0), + ("=INDEX(A1:B3,2,1)", 20.0), + ("=SUM(INDEX(A1:B3,0,1))", 70.0), + ("=IFERROR(INDEX(A1:B3,4,1),99)", 99.0), + ] { + workbook.set_formula(cell(6, 6), formula).unwrap(); + assert_eq!( + workbook.value(cell(6, 6)), + Value::Number(expected), + "{formula}" + ); + } + assert!(matches!( + workbook.set_formula(cell(0, 0), "=INDEX(A1:B3,1,1)"), + Err(FormulaError::Cycle(_)) + )); + let parsed = ParsedFormula::parse("=INDEX(A1:B3,2,1)", 0, &HashMap::new()).unwrap(); + assert_eq!(parsed.reference_count(), 6); + let mapped = parsed.map_references(|id| CellId::new(id.sheet, 2 - id.row, id.column)); + workbook.set_parsed_formula(cell(6, 6), mapped).unwrap(); + assert_eq!(workbook.value(cell(6, 6)), Value::Number(20.0)); + } + #[test] fn reference_envelope_rebinds_and_remains_sparse() { let parsed = ParsedFormula::parse("=SUM(A1:INDEX(A1:A3,2))", 0, &HashMap::new()).unwrap(); diff --git a/crates/omasheets-service/tests/native_controls.rs b/crates/omasheets-service/tests/native_controls.rs index 826ef20..0a3372a 100644 --- a/crates/omasheets-service/tests/native_controls.rs +++ b/crates/omasheets-service/tests/native_controls.rs @@ -9,13 +9,15 @@ struct Fixture { } impl Fixture { fn new() -> Self { + static NEXT_FIXTURE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let path = std::env::temp_dir().join(format!( - "omasheets-controls-{}-{}.omasheets", + "omasheets-controls-{}-{}-{}.omasheets", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + NEXT_FIXTURE.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); let mut f = Self { service: Service::new(|| 456), diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 561d987..874d7ed 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -27,8 +27,9 @@ resolve only to functions already in the registry. selector in D1, and a zero row or column selects that entire axis. The range operator binds a bounded envelope of all possible endpoint selections. Native replay preserves that envelope's row and column identities, including after -sorting. Potential circular dependencies anywhere in the envelope are refused; -this conservative rule is the same as for INDEX's source range. A moved formula +sorting. Constant row/column selections narrow the calculation dependencies after +stable binding, so unused source columns do not create false cycles. Dynamic axes +retain their bounded envelope; potential cycles within it are still refused. A moved formula whose current A1 spelling cannot preserve those identities reports a projection refusal instead of exporting different references. From 1ed0e58e486975ab4e69f014a04665b93f5e9016 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 23:46:57 +0100 Subject: [PATCH 8/9] Add bounded matrix operations and database aggregates --- crates/omasheets-calc/src/database.rs | 163 ++++++++++++++++++++++++++ crates/omasheets-calc/src/lib.rs | 45 ++++++- crates/omasheets-calc/src/matrix.rs | 117 ++++++++++++++++++ crates/omasheets-xlsx/src/lib.rs | 17 +++ docs/FUNCTIONS.md | 23 +++- 5 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 crates/omasheets-calc/src/database.rs create mode 100644 crates/omasheets-calc/src/matrix.rs diff --git a/crates/omasheets-calc/src/database.rs b/crates/omasheets-calc/src/database.rs new file mode 100644 index 0000000..4efc42b --- /dev/null +++ b/crates/omasheets-calc/src/database.rs @@ -0,0 +1,163 @@ +//! Database aggregates share field resolution and row-wise criteria semantics. +use super::*; + +impl Workbook { + pub(super) fn database_aggregate( + &self, + function: Function, + arguments: &[Expr], + ) -> Result { + let [database, field, criteria] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let database = ArrayInput::new(database, self)?; + let criteria = ArrayInput::new(criteria, self)?; + let (rows, columns) = database.shape(); + let (criteria_rows, criteria_columns) = criteria.shape(); + if rows < 2 || criteria_rows < 2 { + return Err(CalcError::InvalidValue); + } + if (rows - 1) + .checked_mul(criteria_rows - 1) + .and_then(|count| count.checked_mul(criteria_columns)) + .is_none_or(|count| count > 50_000_000) + { + return Err(CalcError::InvalidNumber); + } + let header = |name: &str| { + (0..columns).find(|column| matches!(database.value(self, *column), Value::Text(heading) if heading.eq_ignore_ascii_case(name))) + }; + let field = match self.evaluate(field) { + Value::Text(name) => header(&name), + Value::Number(number) + if number.is_finite() && number >= 1.0 && number.trunc() <= columns as f64 => + { + Some(number.trunc() as usize - 1) + } + Value::Error(error) => return Err(error), + _ => None, + } + .ok_or(CalcError::InvalidValue)?; + let fields = (0..criteria_columns) + .map(|column| match criteria.value(self, column) { + Value::Text(name) => header(&name).ok_or(CalcError::InvalidValue), + Value::Error(error) => Err(error), + _ => Err(CalcError::InvalidValue), + }) + .collect::, _>>()?; + let mut filters = Vec::new(); + for row in 1..criteria_rows { + let mut filter = Vec::new(); + for (column, field) in fields.iter().enumerate() { + let mut criterion = criteria.value(self, row * criteria_columns + column); + if matches!(&criterion, Value::Blank) + || matches!(&criterion, Value::Text(text) if text.is_empty()) + { + continue; + } + // Database bare text is a prefix, unlike COUNTIF's exact text. + if let Value::Text(text) = &mut criterion + && !text.starts_with(['=', '<', '>']) + && matches!(parse_criterion(text).1, Value::Text(_)) + { + text.push('*'); + } + filter.push((*field, criterion)); + } + filters.push(filter); + } + let mut values = Vec::new(); + for row in 1..rows { + let mut accepted = false; + for filter in &filters { + let mut matches = true; + for (column, criterion) in filter { + if !criterion_matches( + database.value(self, row * columns + column), + criterion.clone(), + )? { + matches = false; + break; + } + } + if matches { + accepted = true; + break; + } + } + if accepted { + match database.value(self, row * columns + field) { + Value::Number(value) => values.push(value), + Value::Error(error) => return Err(error), + _ => {} + } + } + } + Ok(match function { + Function::DAverage if values.is_empty() => Value::Error(CalcError::DivisionByZero), + Function::DAverage => Value::Number( + values + .iter() + .map(|value| value / values.len() as f64) + .sum::(), + ), + Function::DMin => Value::Number(values.iter().copied().reduce(f64::min).unwrap_or(0.0)), + Function::DMax => Value::Number(values.iter().copied().reduce(f64::max).unwrap_or(0.0)), + Function::DStDev => deviation(&values, true, true), + _ => return Err(CalcError::InvalidArguments), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn database_criteria_combine_rows_columns_prefixes_and_errors() { + let mut workbook = Workbook::default(); + let database = "{\"Tree\",\"Height\",\"Yield\";\"Apple\",18,14;\"Pear\",12,10;\"Cherry\",13,9;\"Apple\",14,10;\"Pear\",9,8;\"Apple\",8,6}"; + for (function, field, criteria, expected) in [ + ( + "DAVERAGE", + "\"Yield\"", + "{\"Tree\",\"Height\";\"=Apple\",\">10\"}", + 12.0, + ), + ("DMAX", "3", "{\"Tree\";\"Pe\"}", 10.0), + ("DMIN", "\"yield\"", "{\"Tree\";\"=?p*\"}", 6.0), + ( + "DAVERAGE", + "3", + "{\"Tree\",\"Height\",\"Height\";\"=Apple\",\">10\",\"<16\";\"=Pear\",\"\",\"\"}", + 28.0 / 3.0, + ), + ("DMAX", "3", "{\"Tree\";\"\"}", 14.0), + ("DMIN", "3", "{\"Tree\";\"=Missing\"}", 0.0), + ("DSTDEV", "3", "{\"Tree\";\"=Pear\"}", 2.0_f64.sqrt()), + ] { + let formula = format!("={function}({database},{field},{criteria})"); + let cell = CellId::new(0, 0, 0); + workbook.set_formula(cell, &formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Number(expected), "{formula}"); + } + for (formula, expected) in [ + ( + "=DAVERAGE({\"A\";1},1,{\"A\";2})", + CalcError::DivisionByZero, + ), + ("=DSTDEV({\"A\";1},1,{\"A\";1})", CalcError::DivisionByZero), + ( + "=DMAX({\"A\";1},1,{\"Expression\";TRUE})", + CalcError::InvalidValue, + ), + ( + "=DMAX({\"A\";#N/A},1,{\"A\";\"\"})", + CalcError::NotAvailable, + ), + ] { + let cell = CellId::new(0, 0, 0); + workbook.set_formula(cell, formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Error(expected), "{formula}"); + } + } +} diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index bc393e2..fd7d09e 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -7,6 +7,8 @@ //! Dates are Excel 1900-system serial numbers; see [`serial_date`] for the //! boundary rules and the deliberately unsupported cases. +mod database; +mod matrix; mod reference; pub mod serial_date; @@ -301,6 +303,12 @@ enum BinaryOp { enum Function { /// Internal reference operator; deliberately absent from the function registry. ReferenceSpan, + Transpose, + MMult, + DAverage, + DMax, + DMin, + DStDev, Sum, Average, Min, @@ -1456,6 +1464,20 @@ impl Workbook { } fn evaluate_function(&self, function: Function, arguments: &[Expr]) -> Value { + if matches!(function, Function::Transpose | Function::MMult) { + return match self.matrix_array(function, arguments) { + Ok(array) => array.values.into_iter().next().unwrap_or(Value::Blank), + Err(error) => Value::Error(error), + }; + } + if matches!( + function, + Function::DAverage | Function::DMax | Function::DMin | Function::DStDev + ) { + return self + .database_aggregate(function, arguments) + .unwrap_or_else(Value::Error); + } if function == Function::ReferenceSpan { return match self.reference_span(arguments) { Ok(reference) => reference.scalar(self), @@ -1806,6 +1828,12 @@ impl Workbook { | Function::HLookup | Function::Row | Function::Column + | Function::Transpose + | Function::MMult + | Function::DAverage + | Function::DMax + | Function::DMin + | Function::DStDev | Function::ReferenceSpan => Value::Error(CalcError::InvalidArguments), } } @@ -2414,6 +2442,9 @@ impl Workbook { fn evaluate_array(&self, expression: &Expr) -> Result { match expression { Expr::Array(array) => Ok(array.clone()), + Expr::Function(function @ (Function::Transpose | Function::MMult), arguments) => { + self.matrix_array(*function, arguments) + } Expr::Function(Function::Index, arguments) => self.index_array(arguments), Expr::Function(Function::ReferenceSpan, arguments) => self .reference_span(arguments) @@ -2861,6 +2892,9 @@ impl<'a> ArrayInput<'a> { columns: *columns, }), Expr::Array(array) => Ok(Self::Constant(array)), + Expr::Function(Function::Transpose | Function::MMult, _) => { + workbook.evaluate_array(expression).map(Self::Computed) + } Expr::Reference(_) | Expr::Function(Function::ReferenceSpan, _) => { workbook.reference_view(expression).map(Self::Selection) } @@ -2947,7 +2981,10 @@ fn contains_array_operand(expression: &Expr) -> bool { match expression { Expr::RangeNode { .. } | Expr::Array(_) - | Expr::Function(Function::Index | Function::ReferenceSpan, _) => true, + | Expr::Function( + Function::Index | Function::ReferenceSpan | Function::Transpose | Function::MMult, + _, + ) => true, Expr::UnaryMinus(inner) | Expr::Percent(inner) => contains_array_operand(inner), Expr::Binary(_, left, right) => { contains_array_operand(left) || contains_array_operand(right) @@ -4717,6 +4754,12 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { /// [`supported_function_names`] both read it, and a test keeps /// `docs/FUNCTIONS.md` in step with it so documented counts cannot drift. const FUNCTION_REGISTRY: &[(&str, Function)] = &[ + ("TRANSPOSE", Function::Transpose), + ("MMULT", Function::MMult), + ("DAVERAGE", Function::DAverage), + ("DMAX", Function::DMax), + ("DMIN", Function::DMin), + ("DSTDEV", Function::DStDev), ("SUM", Function::Sum), ("AVERAGE", Function::Average), ("MIN", Function::Min), diff --git a/crates/omasheets-calc/src/matrix.rs b/crates/omasheets-calc/src/matrix.rs new file mode 100644 index 0000000..96b29c0 --- /dev/null +++ b/crates/omasheets-calc/src/matrix.rs @@ -0,0 +1,117 @@ +//! Bounded matrix operations with explicit shape and numeric-input checks. +use super::*; + +impl Workbook { + pub(super) fn matrix_array( + &self, + function: Function, + arguments: &[Expr], + ) -> Result { + if function == Function::Transpose { + let [input] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let input = self.evaluate_array(input)?; + let mut values = Vec::with_capacity(input.values.len()); + for column in 0..input.columns { + for row in 0..input.rows { + values.push(input.values[row * input.columns + column].clone()); + } + } + return Ok(ArrayValue { + rows: input.columns, + columns: input.rows, + values, + }); + } + let [left, right] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let left = self.evaluate_array(left)?; + let right = self.evaluate_array(right)?; + if left.columns != right.rows { + return Err(CalcError::InvalidValue); + } + let count = left + .rows + .checked_mul(right.columns) + .filter(|count| *count <= MAX_RANGE_CELLS) + .ok_or(CalcError::InvalidNumber)?; + if count + .checked_mul(left.columns) + .is_none_or(|terms| terms > 50_000_000) + { + return Err(CalcError::InvalidNumber); + } + let numbers = |array: ArrayValue| { + array + .values + .into_iter() + .map(|value| match value { + Value::Number(value) if value.is_finite() => Ok(value), + Value::Error(error) => Err(error), + _ => Err(CalcError::InvalidValue), + }) + .collect::, _>>() + }; + let (rows, inner, columns) = (left.rows, left.columns, right.columns); + let left = numbers(left)?; + let right = numbers(right)?; + let mut values = Vec::with_capacity(count); + for row in 0..rows { + for column in 0..columns { + let mut sum = 0.0; + for k in 0..inner { + sum += left[row * inner + k] * right[k * columns + column]; + } + values.push(if sum.is_finite() { + Value::Number(sum) + } else { + Value::Error(CalcError::InvalidNumber) + }); + } + } + Ok(ArrayValue { + rows, + columns, + values, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn matrix_composition_preserves_dimensions_types_and_dependencies() { + let mut workbook = Workbook::default(); + for (formula, expected) in [ + ("=SUM(MMULT({1,2;3,4},{5,6;7,8}))", 134.0), + ("=INDEX(TRANSPOSE({1,2,3;4,5,6}),3,2)", 6.0), + ("=MMULT({1,2,3},TRANSPOSE({4,5,6}))", 32.0), + ("=SUM(TRANSPOSE({1,2;3,4})*{1,2;3,4})", 29.0), + ("=IFERROR(MMULT({1,2},{3,4}),17)", 17.0), + ("=IFERROR(MMULT({1,TRUE},{3;4}),18)", 18.0), + ("=IFERROR(MMULT({1,\"2\"},{3;4}),19)", 19.0), + ("=IFERROR(MMULT({1,#N/A},{3;4}),20)", 20.0), + ("=INDEX(TRANSPOSE({1,\"text\";TRUE,4}),2,2)", 4.0), + ] { + let cell = CellId::new(0, 9, 9); + workbook.set_formula(cell, formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Number(expected), "{formula}"); + } + workbook.set_number(CellId::new(0, 0, 0), 2.0); + workbook.set_number(CellId::new(0, 1, 0), 3.0); + workbook + .set_formula(CellId::new(0, 0, 2), "=MMULT(TRANSPOSE(A1:A2),A1:A2)") + .unwrap(); + assert_eq!(workbook.value(CellId::new(0, 0, 2)), Value::Number(13.0)); + workbook.set_number(CellId::new(0, 1, 0), 4.0); + assert_eq!(workbook.value(CellId::new(0, 0, 2)), Value::Number(20.0)); + workbook.clear(CellId::new(0, 1, 0)); + assert_eq!( + workbook.value(CellId::new(0, 0, 2)), + Value::Error(CalcError::InvalidValue) + ); + } +} diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index 10e4294..629550c 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -1207,6 +1207,23 @@ mod tests { assert_eq!(report.formula_cells_compared, 1); } + #[test] + fn matrix_and_database_formulas_match_independent_caches() { + let path = temporary_xlsx(&package( + &[], + "", + r#"MMULT(TRANSPOSE(A1:A2),A1:A2)13INDEX(TRANSPOSE({1,2,3;4,5,6}),3,2)6DAVERAGE({"Kind","Value";"A",10;"B",50;"A",20},"Value",{"Kind";"=A"})15DMAX({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})20DMIN({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})10DSTDEV({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})7.0710678118654755"#, + "", + )); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); + std::fs::remove_file(path).unwrap(); + assert_eq!(report.formula_cells_loaded, 7); + assert_eq!(report.stored_values_matched, 7); + assert_eq!(report.stored_values_mismatched, 0); + } + #[test] fn reference_valued_index_matches_xlsx_caches() { let bytes = package( diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 874d7ed..01a79da 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -1,7 +1,7 @@ # Supported formula functions The owned M0 engine (`crates/omasheets-calc`) accepts exactly the -104 function names listed below, grouped for reading. +110 function names listed below, grouped for reading. A test in the calc crate fails when this file and the registry disagree, so the count here is never edited by hand: add the function to the registry and regenerate this list. @@ -45,6 +45,27 @@ unsorted keys are undefined in Excel and are not promised here. ## Registry +### Matrices and databases + +- `TRANSPOSE` +- `MMULT` +- `DAVERAGE` +- `DMAX` +- `DMIN` +- `DSTDEV` + +`TRANSPOSE` and `MMULT` produce bounded arrays consumed by aggregates and +lookups; scalar uses take the first element. MMULT requires numeric, nonblank +inputs with matching inner dimensions, at most 1,000,000 output values and +50,000,000 multiply-add terms. Larger products return `#NUM!`. + +Database aggregates resolve fields by heading or one-based column number. +Criteria columns on one row are ANDed; rows are ORed. Duplicate headings, +blank criteria, text prefixes, wildcards and comparison operators are supported. +At most 50,000,000 candidate/criterion comparisons are allowed per call. +Formula criteria with nonmatching/blank headings are not implemented and return +`#VALUE!`; they require relative formula evaluation for each database record. + ### Aggregates and statistics - `SUM` From c3fb774ee84d54acb3e253a1e63c75b69cc4737a Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Tue, 8 Sep 2026 23:46:59 +0100 Subject: [PATCH 9/9] Preserve origin sheet for function-valued range endpoint selectors --- crates/omasheets-calc/src/lib.rs | 21 ++++++++++++++------- crates/omasheets-calc/src/reference.rs | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index fd7d09e..6167fcf 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -4337,15 +4337,22 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { break; } self.offset += 1; - // In Sheet!A1:B2 the unqualified second endpoint inherits Sheet. - let original_sheet = self.sheet; - self.sheet = match &expression { + // Only a bare A1 endpoint inherits the left sheet qualifier. + // Function arguments retain the formula's origin sheet. + self.skip_space(); + let right_start = self.offset; + let mut right = self.parse_primary()?; + let spelling = self.source[right_start..self.offset].trim(); + let inherited_sheet = match &expression { Expr::Reference(cell) | Expr::Range { anchor: cell, .. } => cell.sheet, - _ => reference_bounds(&expression).map_or(self.sheet, |(first, _)| first.sheet), + _ => self.sheet, }; - let right = self.parse_primary(); - self.sheet = original_sheet; - expression = join_reference_range(expression, right?)?; + if matches!(right, Expr::Reference(_)) + && let Ok(cell) = parse_a1(spelling, inherited_sheet) + { + right = Expr::Reference(cell); + } + expression = join_reference_range(expression, right)?; } loop { self.skip_space(); diff --git a/crates/omasheets-calc/src/reference.rs b/crates/omasheets-calc/src/reference.rs index a24b6a0..3b5601f 100644 --- a/crates/omasheets-calc/src/reference.rs +++ b/crates/omasheets-calc/src/reference.rs @@ -422,6 +422,28 @@ mod tests { assert_eq!(workbook.value(cell(5, 6)), Value::Number(70.0)); } + #[test] + fn reference_endpoints_keep_function_selectors_on_the_origin_sheet() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Data"); + workbook.define_sheet(1, "Report"); + for row in 0..3 { + workbook.set_number(CellId::new(0, row, 0), (row + 1) as f64 * 10.0); + } + workbook.set_number(CellId::new(0, 0, 1), 99.0); + workbook.set_number(CellId::new(1, 0, 1), 2.0); + for formula in [ + "=SUM(Data!A1:INDEX(Data!A1:A3,B1))", + "=SUM(INDEX(Data!A1:A3,1):INDEX(Data!A1:A3,B1))", + "=SUM(Data!A1:(INDEX(Data!A1:A3,B1)))", + "=SUM(Data!A1:A2)", + ] { + let target = CellId::new(1, 0, 2); + workbook.set_formula(target, formula).unwrap(); + assert_eq!(workbook.value(target), Value::Number(30.0), "{formula}"); + } + } + #[test] fn constant_index_axes_do_not_create_false_cycles() { let mut workbook = Workbook::default();