diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml new file mode 100644 index 0000000..4d6e0c1 --- /dev/null +++ b/.github/workflows/corpus.yml @@ -0,0 +1,108 @@ +name: Frozen formula corpus + +on: + workflow_dispatch: + inputs: + baseline: + description: Baseline commit or branch to compare on the same runner + required: true + default: main + pull_request: + paths: + - .github/workflows/corpus.yml + +permissions: + contents: read + +concurrency: + group: corpus-${{ github.ref }} + cancel-in-progress: true + +jobs: + compare: + name: Frozen Enron before and after + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CARGO_INCREMENTAL: "0" + steps: + - name: Set runner-local evidence directories + run: | + echo "CORPUS_TARGETS=$RUNNER_TEMP/omasheets-corpus-targets" >> "$GITHUB_ENV" + echo "CORPUS_EVIDENCE=$RUNNER_TEMP/omasheets-corpus-evidence" >> "$GITHUB_ENV" + echo "CORPUS_DATA=$RUNNER_TEMP/omasheets-corpus-data" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: candidate + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha || inputs.baseline }} + path: baseline + persist-credentials: false + - name: Install the minimum toolchain and archive extractor + run: | + sudo apt-get update + sudo apt-get install --yes p7zip-full + rustup toolchain install 1.88.0 --profile minimal --no-self-update + rustup default 1.88.0 + - name: Fetch the registered archive and preserve the frozen denominator + working-directory: candidate + run: | + cmp corpus/sources/enron-figshare.jsonl ../baseline/corpus/sources/enron-figshare.jsonl + cmp corpus/sources/enron-figshare.json ../baseline/corpus/sources/enron-figshare.json + python scripts/fetch_corpus.py corpus/sources/enron-figshare.json "$CORPUS_DATA" + mkdir -p "$CORPUS_EVIDENCE/before" "$CORPUS_EVIDENCE/after" + - name: Build the baseline scorer + working-directory: baseline + run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/baseline" + - name: Verify and score the baseline alone + working-directory: candidate + run: | + "$CORPUS_TARGETS/baseline/release/omasheets-corpus" verify corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" > "$CORPUS_EVIDENCE/verify.json" + python scripts/performance.py run --name frozen-corpus-before --timeout 900 --output "$CORPUS_EVIDENCE/before/performance.json" -- "$CORPUS_TARGETS/baseline/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/before/score.json" --timeout-seconds 30 + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/before/score.json" --performance "$CORPUS_EVIDENCE/before/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary corpus/sources/enron-figshare.score-summary.json --summary "$CORPUS_EVIDENCE/before/summary.json" --delta "$CORPUS_EVIDENCE/before/delta.json" --engine-commit "$(git -C ../baseline rev-parse HEAD)" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured." + - name: Build the candidate scorer + working-directory: candidate + run: cargo build --locked --release --no-default-features -p omasheets-corpus --target-dir "$CORPUS_TARGETS/candidate" + - name: Score the candidate alone and report aggregate changes + working-directory: candidate + run: | + python scripts/performance.py run --name frozen-corpus-after --timeout 900 --output "$CORPUS_EVIDENCE/after/performance.json" -- "$CORPUS_TARGETS/candidate/release/omasheets-corpus" score corpus/sources/enron-figshare.jsonl "$CORPUS_DATA/enron-figshare" "$CORPUS_EVIDENCE/after/score.json" --timeout-seconds 30 + python scripts/update_corpus_summary.py --score "$CORPUS_EVIDENCE/after/score.json" --performance "$CORPUS_EVIDENCE/after/performance.json" --manifest corpus/sources/enron-figshare.jsonl --baseline-summary "$CORPUS_EVIDENCE/before/summary.json" --summary "$CORPUS_EVIDENCE/after/summary.json" --delta "$CORPUS_EVIDENCE/after/delta.json" --runner "GitHub ubuntu-latest x86_64; native-only build; sequential baseline then candidate; hosted runner, not Omarchy hardware" --note "Formualizer candidate lane disabled for this native compatibility iteration; only the owned lane is measured." + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path(os.environ["CORPUS_EVIDENCE"]) + before_report = json.loads((root / "before/summary.json").read_text()) + before = before_report["owned_summary"] + after_report = json.loads((root / "after/summary.json").read_text()) + after = after_report["owned_summary"] + # Retain aggregate evidence in logs if artifact storage is unavailable. + print("FROZEN_CORPUS_BEFORE " + json.dumps(before_report, sort_keys=True)) + print("FROZEN_CORPUS_AFTER " + json.dumps(after_report, sort_keys=True)) + print("FROZEN_CORPUS_DELTA " + (root / "after/delta.json").read_text().replace("\n", " ")) + assert "syntax_failure_tokens" in after, "Candidate scorer lacks the new diagnostic schema" + keys = ["opened", "formula_cells_observed", "formula_cells_loaded", "formula_cells_compared", "stored_values_matched", "stored_values_mismatched", "formula_parse_rate", "stored_value_match_rate"] + lines = ["| Metric | Before | After |", "|---|---:|---:|"] + lines.extend(f"| {key} | {before[key]} | {after[key]} |" for key in keys) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as output: + output.write("\n".join(lines) + "\n") + assert after["opened"] == before["opened"], "Workbook-open denominator changed" + assert after["formula_cells_observed"] == before["formula_cells_observed"], "Formula denominator changed" + assert after["formula_cells_loaded"] >= before["formula_cells_loaded"], "Formula compilation regressed" + assert after["stored_values_matched"] >= before["stored_values_matched"], "Stored-value matches regressed" + PY + - name: Upload aggregate evidence only + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: frozen-formula-corpus + if-no-files-found: warn + path: | + ${{ runner.temp }}/omasheets-corpus-evidence/before/summary.json + ${{ runner.temp }}/omasheets-corpus-evidence/after/summary.json + ${{ runner.temp }}/omasheets-corpus-evidence/after/delta.json diff --git a/corpus/README.md b/corpus/README.md index 54288ce..a61d4af 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -124,6 +124,15 @@ access terms. See `corpus/sources/README.md` for what is registered. ## Corpus policy +The **Frozen formula corpus** Actions workflow scores a baseline and candidate +sequentially on the same hosted runner, in separate build directories, using +the registered archive and the +unchanged 1,000-workbook manifest. Run it manually with a baseline revision; +changes to the workflow itself also exercise it in a pull request. Only +aggregate before/after summaries are uploaded. Syntax failures include a fixed +token-class histogram to guide parser work without exposing formula contents. +This is corpus compatibility evidence, not target-desktop performance evidence. + - Record the upstream corpus name, retrieval date, license or access terms, and sampling method beside every frozen manifest. - Do not commit source workbooks, extracted cell contents, local paths, or model diff --git a/corpus/sources/README.md b/corpus/sources/README.md index d9ca704..20a17ca 100644 --- a/corpus/sources/README.md +++ b/corpus/sources/README.md @@ -28,8 +28,44 @@ the frozen sample for the current engine (engine commit, wall time, process-tree peak memory, both lane summaries, the owned lane's unsupported-function distribution and the failure kinds), and `enron-figshare.score-delta.json` records the owned lane against the -baseline engine that first scored the sample. Both are aggregate only. Scored -on 2026-09-01 on a 4-vCPU Linux container: +baseline engine named in that delta. Both are aggregate only; the JSON records +the exact engine revisions and runners. + +The latest comparison was scored on 2026-09-08, sequentially on one GitHub +`ubuntu-latest` runner with separate build directories. Both revisions used +the same frozen 1,000-workbook manifest and exposed 924,235 formula cells. +[Workflow evidence](https://github.com/tcballard/OmaSheets/actions/runs/34254502652). + +| Owned engine lane | Baseline (`ecf0036`) | Formula coverage (`12d1c8c`) | +|---|---:|---:| +| Workbooks opened | 996 / 1,000 | 996 / 1,000 | +| Formula cells observed | 924,235 | 924,235 | +| Loaded and compared | 825,016 (89.26%) | 829,529 (89.75%) | +| Stored values matched | 823,932 | 828,437 | +| Match rate of compared | 99.87% | 99.87% | +| Stored values mismatched | 1,084 | 1,092 | +| Not compiled | 99,219 | 94,706 | + +This adds 4,513 compiled formulas and 4,505 stored-value matches without +changing the sample, comparison tolerance or denominator. The eight additional +mismatches remain visible. The aggregate cannot attribute them to individual +formulas or establish that every previously matched cell is unchanged. + +Parser work also reveals later failures: the 10,791 syntax classifications +are gone, while invalid references rise from 4,260 to 10,551 and unsupported +function classifications rise from 26,617 to 28,967. `OFFSET` now accounts for +3,619 first failures; previously some of these stopped at a syntax or name +error. A lower count in an early failure class does not mean every affected +formula now compiles. + +The remaining first-failure groups are 47,007 external workbook references, +28,967 unsupported functions, 10,551 invalid references, 5,767 unsupported +name definitions, 2,292 cycles and 122 unknown names. Within unsupported +functions, 21,737 cells call proprietary add-ins. Ordinary gaps include +`OFFSET`, locale-sensitive `DATEVALUE`, volatile functions and `INDIRECT`. +External/add-in execution and hidden clock or random state remain refused. +These are compatibility measurements, not target-desktop performance claims. + Generate both files from a measured schema-2 scorer report with `scripts/update_corpus_summary.py`. The command validates the manifest digest, @@ -49,6 +85,12 @@ python scripts/update_corpus_summary.py \ --runner "linux x86_64, eight-core baseline, cold run" ``` +### Historical comparison + +The following 2026-09-01 run predates importer fixes that exposed additional +formula cells. Its smaller denominator makes the raw rate unsuitable for a +direct comparison with the current table. + | Owned M0 engine lane | Baseline (`08f38d1`) | After formula gaps (`0eccaff`) | |---|---|---| | Workbooks opened | 971 / 1000 | 971 / 1000 | diff --git a/corpus/sources/enron-figshare.score-delta.json b/corpus/sources/enron-figshare.score-delta.json index 598b028..85eb996 100644 --- a/corpus/sources/enron-figshare.score-delta.json +++ b/corpus/sources/enron-figshare.score-delta.json @@ -1,86 +1,108 @@ { - "schema": 1, - "source": "enron-figshare", + "baseline_engine_commit": "ecf00367004ccbeb56a736fd0aef30dd4e45c1ba", + "engine_commit": "12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5", "manifest": "enron-figshare.jsonl", "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", - "baseline_engine_commit": "3d32520f91d51b63d327202cca82920125e7d36d", - "engine_commit": "71f21c5e285c7ec98543b498088c17ee648ef858", - "scored": "2026-09-02", + "notes": [ + "Frozen 1,000-workbook manifest and 924,235-formula denominator unchanged.", + "Workflow evidence: https://github.com/tcballard/OmaSheets/actions/runs/34254502652", + "First-failure categories move as parsing advances: syntax 10,791 -> 0, invalid references 4,260 -> 10,551, unsupported functions 26,617 -> 28,967. This is not a claim that every former syntax failure now evaluates.", + "4,513 more formulas compiled; 4,505 more stored-value matches; total mismatches 1,084 -> 1,092. Aggregate evidence cannot assign the eight additional mismatches to individual formulas.", + "No performance comparison to Omarchy desktop hardware is implied." + ], "owned_lane": { - "opened": { - "before": 996, - "after": 996 + "comparison_coverage": { + "after": 0.8975303899982148, + "before": 0.8926474327416728 }, "failed": { - "before": 4, - "after": 4 - }, - "timed_out": { - "before": 0, - "after": 0 + "after": 4, + "before": 4 }, - "formula_cells_observed": { - "before": 924235, - "after": 924235 + "formula_cells_compared": { + "after": 829529, + "before": 825016 }, "formula_cells_loaded": { - "before": 817225, - "after": 825016 + "after": 829529, + "before": 825016 }, - "formula_cells_compared": { - "before": 817225, - "after": 825016 + "formula_cells_observed": { + "after": 924235, + "before": 924235 }, - "stored_values_matched": { - "before": 816141, - "after": 823932 + "formula_parse_rate": { + "after": 0.8975303899982148, + "before": 0.8926474327416728 }, - "stored_values_mismatched": { - "before": 1084, - "after": 1084 + "opened": { + "after": 996, + "before": 996 }, - "unsupported_formulas": { - "before": 107010, - "after": 99219 + "peak_rss_bytes_max": { + "after": 142831616, + "before": 142925824 }, - "formula_parse_rate": { - "before": 0.8842177584705189, - "after": 0.8926474327416728 + "stored_value_match_rate": { + "after": 0.9986835903265588, + "before": 0.9986860860880273 }, - "comparison_coverage": { - "before": 0.8842177584705189, - "after": 0.8926474327416728 + "stored_values_matched": { + "after": 828437, + "before": 823932 }, - "stored_value_match_rate": { - "before": 0.9986735599131206, - "after": 0.9986860860880273 + "stored_values_mismatched": { + "after": 1092, + "before": 1084 }, - "peak_rss_bytes_max": { - "before": 116269056, - "after": 116408320 + "timed_out": { + "after": 0, + "before": 0 + }, + "unsupported_formulas": { + "after": 94706, + "before": 99219 } }, "owned_lane_added": { - "workbooks_with_skipped_sheets": { - "before": 23, - "after": 23 - }, "skipped_sheets": { - "before": 48, - "after": 48 + "after": 48, + "before": 48 + }, + "workbooks_with_skipped_sheets": { + "after": 23, + "before": 23 } }, + "process_tree_peak_rss_bytes": { + "after": 1493118976, + "before": 1502441472 + }, + "resolved_defects": [ + "Array constants in aggregate, elementwise and lookup inputs", + "Sheet-qualified defined names and known modern function prefixes", + "Qualified and deleted reference range endpoints" + ], + "resolved_functions": [ + "COVAR", + "IRR", + "NORMSDIST", + "PV" + ], + "resolved_open_failures": {}, + "schema": 1, + "scored": "2026-09-08", + "source": "enron-figshare", "unsupported_reasons": { - "before": { - "cycle": 2286, + "after": { + "cycle": 2292, "external_reference": 47007, - "invalid_reference": 12051, - "syntax": 10791, + "invalid_reference": 10551, "unknown_name": 122, - "unsupported_function": 26617, - "unsupported_name": 8136 + "unsupported_function": 28967, + "unsupported_name": 5767 }, - "after": { + "before": { "cycle": 2286, "external_reference": 47007, "invalid_reference": 4260, @@ -90,20 +112,8 @@ "unsupported_name": 8136 } }, - "resolved_functions": [], - "resolved_defects": [ - "a sheet-qualified #REF! (a deleted cell on another sheet) was refused as an invalid reference instead of compiling to the #REF! error" - ], - "resolved_open_failures": {}, "wall_seconds": { - "before": 139.0, - "after": 208.0 - }, - "process_tree_peak_rss_bytes": { - "before": 1539121152, - "after": 1544372224 - }, - "notes": [ - "Timing evidence in this entry comes from a restarted container that is about 35% slower than the one the previous entries ran on: the candidate lane, which this engine change does not touch, went from 44 s to 60 s of summed import time. Per-cell results are identical across two runs on the new container; compare cell counts across entries, not wall seconds." - ] + "after": 117.606367, + "before": 118.763427 + } } diff --git a/corpus/sources/enron-figshare.score-summary.json b/corpus/sources/enron-figshare.score-summary.json index e23fc80..9694c01 100644 --- a/corpus/sources/enron-figshare.score-summary.json +++ b/corpus/sources/enron-figshare.score-summary.json @@ -1,53 +1,64 @@ { - "schema": 1, - "source": "enron-figshare", - "manifest": "enron-figshare.jsonl", - "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", - "scored": "2026-09-02", - "command": "omasheets-corpus score corpus/sources/enron-figshare.jsonl --timeout-seconds 30", - "engine_commit": "71f21c5e285c7ec98543b498088c17ee648ef858", - "runner": "linux x86_64 container, 4 vCPU", - "wall_seconds": 208.0, - "process_tree_peak_rss_bytes": 1544372224, "candidate_engine": "formualizer-calamine-0.8.4", + "candidate_failure_kinds": { + "chartsheet_instead_of_worksheet": 39, + "memory_allocation_failed": 2, + "other": 2, + "relationship_not_found": 24, + "undefined_name_or_formula_parse": 99 + }, "candidate_summary": { + "failed": 166, "files": 1000, + "formula_cells_loaded": 419854, + "formula_cells_observed": 419854, + "formula_parse_rate": 1, + "peak_rss_bytes_max": 154779648, "succeeded": 834, - "failed": 166, "timed_out": 0, - "value_cells_observed": 3843371, - "formula_cells_observed": 419854, - "formula_cells_loaded": 419854, - "formula_parse_rate": 1.0, - "peak_rss_bytes_max": 160321536 + "value_cells_observed": 3843371 }, + "command": "omasheets-corpus score enron-figshare.jsonl --timeout-seconds 30", + "engine_commit": "12d1c8c69522ee7ee81ce3d7090a1bba60cd2ac5", + "manifest": "enron-figshare.jsonl", + "manifest_sha256": "2f5ea94c02ae3779206d6ca1b1da9e77002450bc070fa968ca566cfd526ff80d", + "notes": [ + "Aggregate only: no workbook paths, cell contents or per-file results are recorded here.", + "Baseline and candidate scored sequentially in isolated build directories on the same GitHub runner; unchanged frozen 1,000-workbook manifest.", + "Workflow evidence: https://github.com/tcballard/OmaSheets/actions/runs/34254502652", + "Raw coverage retains external references, proprietary add-ins, cycles and unsupported formulas in the denominator." + ], "owned_engine": "omasheets-owned-m0", + "owned_failure_kinds": { + "cell_limit_2000000_exceeded": 2, + "date_system_1904_rejected": 2 + }, "owned_summary": { + "comparison_coverage": 0.8975303899982148, + "failed": 4, "files": 1000, + "formula_cells_compared": 829529, + "formula_cells_loaded": 829529, + "formula_cells_observed": 924235, + "formula_parse_rate": 0.8975303899982148, "opened": 996, - "failed": 4, + "peak_rss_bytes_max": 142831616, + "skipped_sheets": 48, + "stored_value_match_rate": 0.9986835903265588, + "stored_values_matched": 828437, + "stored_values_mismatched": 1092, + "syntax_failure_tokens": {}, "timed_out": 0, - "formula_cells_observed": 924235, - "formula_cells_loaded": 825016, - "formula_cells_compared": 825016, - "stored_values_matched": 823932, - "stored_values_mismatched": 1084, - "unsupported_formulas": 99219, - "formula_parse_rate": 0.8926474327416728, - "comparison_coverage": 0.8926474327416728, - "stored_value_match_rate": 0.9986860860880273, + "unsupported_formulas": 94706, "unsupported_reasons": { - "cycle": 2286, + "cycle": 2292, "external_reference": 47007, - "invalid_reference": 4260, - "syntax": 10791, + "invalid_reference": 10551, "unknown_name": 122, - "unsupported_function": 26617, - "unsupported_name": 8136 + "unsupported_function": 28967, + "unsupported_name": 5767 }, - "workbooks_with_skipped_sheets": 23, - "skipped_sheets": 48, - "peak_rss_bytes_max": 116408320 + "workbooks_with_skipped_sheets": 23 }, "owned_unsupported_functions": { "_XLL.HPVAL": { @@ -62,6 +73,10 @@ "formula_cells": 3642, "workbooks": 1 }, + "OFFSET": { + "formula_cells": 3619, + "workbooks": 10 + }, "_XLL.EURO": { "formula_cells": 2992, "workbooks": 2 @@ -74,10 +89,6 @@ "formula_cells": 1412, "workbooks": 2 }, - "OFFSET": { - "formula_cells": 1250, - "workbooks": 10 - }, "CALCSKEW": { "formula_cells": 1177, "workbooks": 1 @@ -162,26 +173,10 @@ "formula_cells": 10, "workbooks": 1 }, - "IRR": { - "formula_cells": 6, - "workbooks": 1 - }, - "PV": { - "formula_cells": 6, - "workbooks": 3 - }, "MMULT": { "formula_cells": 4, "workbooks": 1 }, - "NORMSDIST": { - "formula_cells": 4, - "workbooks": 1 - }, - "COVAR": { - "formula_cells": 3, - "workbooks": 1 - }, "_XLL.XASTRIP": { "formula_cells": 2, "workbooks": 1 @@ -195,28 +190,10 @@ "workbooks": 1 } }, - "candidate_failure_kinds": { - "undefined_name_or_formula_parse": 99, - "chartsheet_instead_of_worksheet": 40, - "relationship_not_found": 24, - "memory_allocation_failed": 2, - "other": 1 - }, - "owned_failure_kinds": { - "date_system_1904_rejected": 2, - "cell_limit_2000000_exceeded": 2 - }, - "notes": [ - "Aggregate only: no workbook paths, cell contents or per-file results are recorded here.", - "The owned importer opens 23 workbooks whose xl/workbook.xml lists veryHidden sheet entries with no worksheet part (converted legacy macro modules) by skipping those entries in an in-memory copy; the 48 skipped names are reported per workbook, never silently.", - "The four remaining owned-lane failures are two 1904-date-system workbooks, refused by decision (ADR-0004), and two workbooks over the 2,000,000-cell import bound.", - "external_reference counts formulas that reach into another workbook; the engine never evaluates them.", - "Ranges are shared graph nodes with membership decided by position, and imports load in bulk with one recalculation; the worst-workbook peak resident set fell from about 1 GB to about 110 MB and the two former 30 s timeouts import in well under 10 s.", - "Twelve functions the corpus used most without support (YEARFRAC, DAYS360, NETWORKDAYS, WORKDAY, LOOKUP, PMT, NPV, XNPV, XIRR, NORMDIST, AVERAGEA, CORREL) now evaluate; two workbooks whose defined names are literally #REF! now report #REF! where Excel had kept a stale cached value.", - "Mismatch round 1: tokens past the grid resolve as names, sheet-scoped names keep their scope, criteria follow Excel for spacing, wildcards, blanks, errors and mixed types, blank lookups never match blank cells, negative zero folds to zero, INDEX passes a failed MATCH through, lookups skip error cells, math domain errors are #NUM!, and shared formulas expand from their anchor cell; the eight worst-mismatch workbooks of the previous score now match Excel cell for cell apart from three macro-backed cells.", - "Mismatch round 2: numbers compare at 15 significant digits as in Excel, and range expressions inside aggregate arguments (SUM(IF(...)), SUM(A:A*B:B), SUMPRODUCT((...)*...)) evaluate elementwise with Excel's broadcasting.", - "Sheet-qualified #REF! tokens compile to the #REF! error value, which is what Excel stores for a deleted cell on another sheet.", - "Timing evidence in this entry comes from a restarted container that is about 35% slower than the one the previous entries ran on: the candidate lane, which this engine change does not touch, went from 44 s to 60 s of summed import time. Per-cell results are identical across two runs on the new container; compare cell counts across entries, not wall seconds.", - "See enron-figshare.score-delta.json for the change against the previous engine." - ] + "process_tree_peak_rss_bytes": 1493118976, + "runner": "GitHub ubuntu-latest x86_64; sequential baseline then candidate; hosted runner, not Omarchy hardware", + "schema": 1, + "scored": "2026-09-08", + "source": "enron-figshare", + "wall_seconds": 117.606367 } diff --git a/crates/omasheets-calc/src/database.rs b/crates/omasheets-calc/src/database.rs new file mode 100644 index 0000000..4efc42b --- /dev/null +++ b/crates/omasheets-calc/src/database.rs @@ -0,0 +1,163 @@ +//! Database aggregates share field resolution and row-wise criteria semantics. +use super::*; + +impl Workbook { + pub(super) fn database_aggregate( + &self, + function: Function, + arguments: &[Expr], + ) -> Result { + let [database, field, criteria] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let database = ArrayInput::new(database, self)?; + let criteria = ArrayInput::new(criteria, self)?; + let (rows, columns) = database.shape(); + let (criteria_rows, criteria_columns) = criteria.shape(); + if rows < 2 || criteria_rows < 2 { + return Err(CalcError::InvalidValue); + } + if (rows - 1) + .checked_mul(criteria_rows - 1) + .and_then(|count| count.checked_mul(criteria_columns)) + .is_none_or(|count| count > 50_000_000) + { + return Err(CalcError::InvalidNumber); + } + let header = |name: &str| { + (0..columns).find(|column| matches!(database.value(self, *column), Value::Text(heading) if heading.eq_ignore_ascii_case(name))) + }; + let field = match self.evaluate(field) { + Value::Text(name) => header(&name), + Value::Number(number) + if number.is_finite() && number >= 1.0 && number.trunc() <= columns as f64 => + { + Some(number.trunc() as usize - 1) + } + Value::Error(error) => return Err(error), + _ => None, + } + .ok_or(CalcError::InvalidValue)?; + let fields = (0..criteria_columns) + .map(|column| match criteria.value(self, column) { + Value::Text(name) => header(&name).ok_or(CalcError::InvalidValue), + Value::Error(error) => Err(error), + _ => Err(CalcError::InvalidValue), + }) + .collect::, _>>()?; + let mut filters = Vec::new(); + for row in 1..criteria_rows { + let mut filter = Vec::new(); + for (column, field) in fields.iter().enumerate() { + let mut criterion = criteria.value(self, row * criteria_columns + column); + if matches!(&criterion, Value::Blank) + || matches!(&criterion, Value::Text(text) if text.is_empty()) + { + continue; + } + // Database bare text is a prefix, unlike COUNTIF's exact text. + if let Value::Text(text) = &mut criterion + && !text.starts_with(['=', '<', '>']) + && matches!(parse_criterion(text).1, Value::Text(_)) + { + text.push('*'); + } + filter.push((*field, criterion)); + } + filters.push(filter); + } + let mut values = Vec::new(); + for row in 1..rows { + let mut accepted = false; + for filter in &filters { + let mut matches = true; + for (column, criterion) in filter { + if !criterion_matches( + database.value(self, row * columns + column), + criterion.clone(), + )? { + matches = false; + break; + } + } + if matches { + accepted = true; + break; + } + } + if accepted { + match database.value(self, row * columns + field) { + Value::Number(value) => values.push(value), + Value::Error(error) => return Err(error), + _ => {} + } + } + } + Ok(match function { + Function::DAverage if values.is_empty() => Value::Error(CalcError::DivisionByZero), + Function::DAverage => Value::Number( + values + .iter() + .map(|value| value / values.len() as f64) + .sum::(), + ), + Function::DMin => Value::Number(values.iter().copied().reduce(f64::min).unwrap_or(0.0)), + Function::DMax => Value::Number(values.iter().copied().reduce(f64::max).unwrap_or(0.0)), + Function::DStDev => deviation(&values, true, true), + _ => return Err(CalcError::InvalidArguments), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn database_criteria_combine_rows_columns_prefixes_and_errors() { + let mut workbook = Workbook::default(); + let database = "{\"Tree\",\"Height\",\"Yield\";\"Apple\",18,14;\"Pear\",12,10;\"Cherry\",13,9;\"Apple\",14,10;\"Pear\",9,8;\"Apple\",8,6}"; + for (function, field, criteria, expected) in [ + ( + "DAVERAGE", + "\"Yield\"", + "{\"Tree\",\"Height\";\"=Apple\",\">10\"}", + 12.0, + ), + ("DMAX", "3", "{\"Tree\";\"Pe\"}", 10.0), + ("DMIN", "\"yield\"", "{\"Tree\";\"=?p*\"}", 6.0), + ( + "DAVERAGE", + "3", + "{\"Tree\",\"Height\",\"Height\";\"=Apple\",\">10\",\"<16\";\"=Pear\",\"\",\"\"}", + 28.0 / 3.0, + ), + ("DMAX", "3", "{\"Tree\";\"\"}", 14.0), + ("DMIN", "3", "{\"Tree\";\"=Missing\"}", 0.0), + ("DSTDEV", "3", "{\"Tree\";\"=Pear\"}", 2.0_f64.sqrt()), + ] { + let formula = format!("={function}({database},{field},{criteria})"); + let cell = CellId::new(0, 0, 0); + workbook.set_formula(cell, &formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Number(expected), "{formula}"); + } + for (formula, expected) in [ + ( + "=DAVERAGE({\"A\";1},1,{\"A\";2})", + CalcError::DivisionByZero, + ), + ("=DSTDEV({\"A\";1},1,{\"A\";1})", CalcError::DivisionByZero), + ( + "=DMAX({\"A\";1},1,{\"Expression\";TRUE})", + CalcError::InvalidValue, + ), + ( + "=DMAX({\"A\";#N/A},1,{\"A\";\"\"})", + CalcError::NotAvailable, + ), + ] { + let cell = CellId::new(0, 0, 0); + workbook.set_formula(cell, formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Error(expected), "{formula}"); + } + } +} diff --git a/crates/omasheets-calc/src/lib.rs b/crates/omasheets-calc/src/lib.rs index fe287f0..6167fcf 100644 --- a/crates/omasheets-calc/src/lib.rs +++ b/crates/omasheets-calc/src/lib.rs @@ -7,6 +7,9 @@ //! Dates are Excel 1900-system serial numbers; see [`serial_date`] for the //! boundary rules and the deliberately unsupported cases. +mod database; +mod matrix; +mod reference; pub mod serial_date; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; @@ -191,6 +194,8 @@ enum Expr { Error(CalcError), /// An omitted argument, as in `IF(x,,y)`; evaluates to blank. Empty, + /// A bounded rectangular array of literal values, with no dependencies. + Array(ArrayValue), Reference(R), UnaryMinus(Box>), Percent(Box>), @@ -296,6 +301,14 @@ enum BinaryOp { #[derive(Clone, Copy, Debug, PartialEq)] enum Function { + /// Internal reference operator; deliberately absent from the function registry. + ReferenceSpan, + Transpose, + MMult, + DAverage, + DMax, + DMin, + DStDev, Sum, Average, Min, @@ -359,10 +372,16 @@ enum Function { WorkDay, Lookup, Pmt, + Pv, + Irr, Npv, Xnpv, Xirr, NormDist, + NormSDist, + NormSDistLegacy, + CovarianceP, + CovarianceS, AverageA, Correl, IsBlank, @@ -548,6 +567,7 @@ fn visit_references(expression: &Expr, visit: &mut impl FnMut(CellId)) { | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -594,6 +614,7 @@ fn rebind_references(expression: &mut Expr, map: &mut impl FnMut(CellId) | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -794,7 +815,7 @@ impl Workbook { cell: CellId, formula: ParsedFormula, ) -> Result { - let parsed = formula.expression; + let parsed = reference::narrow_reference_dependencies(formula.expression); let mut cells = BTreeSet::new(); let mut range_keys = Vec::new(); collect_dependencies(&parsed, &mut cells, &mut range_keys); @@ -1019,13 +1040,6 @@ impl Workbook { statistics } - fn range_len(&self, node: usize) -> usize { - match self.range_shape(node) { - RangeShape::Rectangle { rows, columns, .. } => rows * columns, - RangeShape::Members { .. } => self.cells[node].dependencies.len(), - } - } - /// The cell at a row-major position inside the range, if it exists. fn range_cell(&self, node: usize, index: usize) -> Option { match self.range_shape(node) { @@ -1378,6 +1392,7 @@ impl Workbook { Expr::Text(value) => Value::Text(value.clone()), Expr::Error(error) => Value::Error(error.clone()), Expr::Empty => Value::Blank, + Expr::Array(array) => array.values[0].clone(), Expr::Reference(index) => self.cells[*index].value.clone(), Expr::UnaryMinus(inner) => match self.evaluate(inner) { Value::Number(value) => number_value(-value), @@ -1449,6 +1464,26 @@ impl Workbook { } fn evaluate_function(&self, function: Function, arguments: &[Expr]) -> Value { + if matches!(function, Function::Transpose | Function::MMult) { + return match self.matrix_array(function, arguments) { + Ok(array) => array.values.into_iter().next().unwrap_or(Value::Blank), + Err(error) => Value::Error(error), + }; + } + if matches!( + function, + Function::DAverage | Function::DMax | Function::DMin | Function::DStDev + ) { + return self + .database_aggregate(function, arguments) + .unwrap_or_else(Value::Error); + } + if function == Function::ReferenceSpan { + return match self.reference_span(arguments) { + Ok(reference) => reference.scalar(self), + Err(error) => Value::Error(error), + }; + } if function == Function::If { if !matches!(arguments.len(), 2 | 3) { return Value::Error(CalcError::InvalidArguments); @@ -1597,12 +1632,20 @@ impl Workbook { } if matches!( function, - Function::Pmt | Function::Npv | Function::Xnpv | Function::Xirr + Function::Pmt + | Function::Pv + | Function::Irr + | Function::Npv + | Function::Xnpv + | Function::Xirr ) { return self.evaluate_financial_function(function, arguments); } - if function == Function::Correl { - return self.evaluate_correl(arguments); + if matches!( + function, + Function::Correl | Function::CovarianceP | Function::CovarianceS + ) { + return self.evaluate_paired_statistics(function, arguments); } if matches!( @@ -1639,6 +1682,22 @@ impl Workbook { match function { Function::Median if !numbers.is_empty() => median(numbers), Function::NormDist => normal_distribution(&values), + Function::NormSDist | Function::NormSDistLegacy => { + let required = if function == Function::NormSDistLegacy { + 1 + } else { + 2 + }; + if values.len() != required { + return Value::Error(CalcError::InvalidArguments); + } + normal_distribution(&[ + values[0].clone(), + Value::Number(0.0), + Value::Number(1.0), + values.get(1).cloned().unwrap_or(Value::Boolean(true)), + ]) + } Function::AverageA => average_a(&values), Function::StDev => deviation(&numbers, true, true), Function::StDevP => deviation(&numbers, false, true), @@ -1737,10 +1796,14 @@ impl Workbook { | Function::NetworkDays | Function::WorkDay | Function::Pmt + | Function::Pv + | Function::Irr | Function::Npv | Function::Xnpv | Function::Xirr | Function::Correl + | Function::CovarianceP + | Function::CovarianceS | Function::Date | Function::Year | Function::Month @@ -1764,7 +1827,14 @@ impl Workbook { | Function::IsNa | Function::HLookup | Function::Row - | Function::Column => Value::Error(CalcError::InvalidArguments), + | Function::Column + | Function::Transpose + | Function::MMult + | Function::DAverage + | Function::DMax + | Function::DMin + | Function::DStDev + | Function::ReferenceSpan => Value::Error(CalcError::InvalidArguments), } } @@ -2088,39 +2158,42 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; + let (rows, columns) = input.shape(); let (candidates, results): (Vec, Vec) = if arguments.len() == 3 { - let Some((result_node, result_rows, result_columns)) = range_parts(&arguments[2]) - else { - return Value::Error(CalcError::InvalidArguments); + let result_input = match ArrayInput::new(&arguments[2], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; + let (result_rows, result_columns) = result_input.shape(); if (rows != 1 && columns != 1) || (result_rows != 1 && result_columns != 1) || rows * columns != result_rows * result_columns { return Value::Error(CalcError::InvalidArguments); } - (self.range_values(node), self.range_values(result_node)) + (input.values(self), result_input.values(self)) } else if rows == 1 || columns == 1 { - let values = self.range_values(node); + let values = input.values(self); (values.clone(), values) } else if columns > rows { // More columns than rows: search the first row, answer from the last. let candidates = (0..columns) - .map(|column| self.range_value(node, column)) + .map(|column| input.value(self, column)) .collect(); let results = (0..columns) - .map(|column| self.range_value(node, (rows - 1) * columns + column)) + .map(|column| input.value(self, (rows - 1) * columns + column)) .collect(); (candidates, results) } else { let candidates = (0..rows) - .map(|row| self.range_value(node, row * columns)) + .map(|row| input.value(self, row * columns)) .collect(); let results = (0..rows) - .map(|row| self.range_value(node, row * columns + columns - 1)) + .map(|row| input.value(self, row * columns + columns - 1)) .collect(); (candidates, results) }; @@ -2130,10 +2203,10 @@ impl Workbook { } } - /// `PMT`, `NPV`, `XNPV` and `XIRR`. + /// Present values, annuity payments, and periodic or dated returns. fn evaluate_financial_function(&self, function: Function, arguments: &[Expr]) -> Value { let result = match function { - Function::Pmt => { + Function::Pmt | Function::Pv => { if !matches!(arguments.len(), 3..=5) { return Value::Error(CalcError::InvalidArguments); } @@ -2144,7 +2217,12 @@ impl Workbook { Err(error) => return Value::Error(error), } } - payment( + let calculate = if function == Function::Pv { + present_value + } else { + payment + }; + calculate( numbers[0], numbers[1], numbers[2], @@ -2152,6 +2230,22 @@ impl Workbook { numbers.get(4).copied().unwrap_or(0.0) != 0.0, ) } + Function::Irr => { + if !matches!(arguments.len(), 1 | 2) { + return Value::Error(CalcError::InvalidArguments); + } + let mut values = Vec::new(); + self.flatten_values(&arguments[0], &mut values); + if let Some(error) = first_error(&values) { + return Value::Error(error); + } + let guess = arguments + .get(1) + .map_or(Ok(0.1), |argument| number(self.evaluate(argument))); + guess.and_then(|guess| { + periodic_internal_rate_of_return(&numeric_only(&values), guess) + }) + } Function::Npv => { if arguments.len() < 2 { return Value::Error(CalcError::InvalidArguments); @@ -2209,9 +2303,8 @@ impl Workbook { } } - /// `CORREL(array1, array2)`: Pearson correlation over the positions where - /// both sides are numbers. - fn evaluate_correl(&self, arguments: &[Expr]) -> Value { + /// Correlation or covariance over positions where both sides are numbers. + fn evaluate_paired_statistics(&self, function: Function, arguments: &[Expr]) -> Value { if arguments.len() != 2 { return Value::Error(CalcError::InvalidArguments); } @@ -2233,7 +2326,12 @@ impl Workbook { _ => None, }) .collect(); - match correlation(&pairs) { + let result = if function == Function::Correl { + correlation(&pairs) + } else { + covariance(&pairs, function == Function::CovarianceS) + }; + match result { Ok(value) => Value::Number(value), Err(error) => Value::Error(error), } @@ -2288,6 +2386,16 @@ impl Workbook { } } }, + Expr::Function(Function::ReferenceSpan | Function::Index, _) => { + match self.reference_view(argument) { + Ok(reference) => reference.visit(self, &mut visit), + Err(CalcError::InvalidArguments) => match self.evaluate_array(argument) { + Ok(array) => array.values.iter().for_each(&mut visit), + Err(error) => visit(&Value::Error(error)), + }, + Err(error) => visit(&Value::Error(error)), + } + } Expr::Empty => {} other if contains_array_operand(other) => match self.evaluate_array(other) { Ok(array) => array.values.iter().for_each(&mut visit), @@ -2333,6 +2441,14 @@ impl Workbook { /// `#N/A`. Bounded by [`MAX_RANGE_CELLS`]. fn evaluate_array(&self, expression: &Expr) -> Result { match expression { + Expr::Array(array) => Ok(array.clone()), + Expr::Function(function @ (Function::Transpose | Function::MMult), arguments) => { + self.matrix_array(*function, arguments) + } + Expr::Function(Function::Index, arguments) => self.index_array(arguments), + Expr::Function(Function::ReferenceSpan, arguments) => self + .reference_span(arguments) + .map(|reference| reference.array(self)), Expr::RangeNode { node, rows, @@ -2502,34 +2618,17 @@ impl Workbook { fn evaluate_lookup_function(&self, function: Function, arguments: &[Expr]) -> Value { match function { Function::Index if matches!(arguments.len(), 2 | 3) => { - let Some((node, rows, columns)) = range_parts(&arguments[0]) else { - return Value::Error(CalcError::InvalidArguments); - }; - // An error row (a failed MATCH) is the result, as in Excel. - let row = match positive_index(self.evaluate(&arguments[1])) { - Ok(row) => row, - Err(error) => return Value::Error(error), - }; - let column = if arguments.len() == 3 { - match positive_index(self.evaluate(&arguments[2])) { - Ok(column) => column, + if !matches!(arguments[0], Expr::Array(_)) { + match self.reference_view(&Expr::Function(function, arguments.to_vec())) { + Ok(reference) => return reference.scalar(self), + Err(CalcError::InvalidArguments) => {} Err(error) => return Value::Error(error), } - } else if columns == 1 { - 1 - } else if rows == 1 { - return if row <= columns { - self.range_value(node, row - 1) - } else { - Value::Error(CalcError::InvalidReference) - }; - } else { - return Value::Error(CalcError::InvalidArguments); - }; - if row > rows || column > columns { - return Value::Error(CalcError::InvalidReference); } - self.range_value(node, (row - 1) * columns + column - 1) + match self.index_array(arguments) { + Ok(array) => array.values[0].clone(), + Err(error) => Value::Error(error), + } } Function::Match if matches!(arguments.len(), 2 | 3) => { let mode = if arguments.len() == 3 { @@ -2544,13 +2643,15 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; + let (rows, columns) = input.shape(); if rows != 1 && columns != 1 { return Value::Error(CalcError::InvalidArguments); } - let candidates = self.range_values(node); + let candidates = input.values(self); match match_position(&lookup, &candidates, mode) { Ok(position) => Value::Number((position + 1) as f64), Err(error) => Value::Error(error), @@ -2577,9 +2678,11 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((node, rows, columns)) = range_parts(&arguments[1]) else { - return Value::Error(CalcError::InvalidArguments); + let input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; + let (rows, columns) = input.shape(); let Ok(offset) = positive_index(self.evaluate(&arguments[2])) else { return Value::Error(CalcError::InvalidValue); }; @@ -2594,9 +2697,9 @@ impl Workbook { } let at = |lane: usize, position: usize| { if vertical { - self.range_value(node, lane * columns + position) + input.value(self, lane * columns + position) } else { - self.range_value(node, position * columns + lane) + input.value(self, position * columns + lane) } }; let candidates: Vec = (0..lanes).map(|lane| at(lane, 0)).collect(); @@ -2610,25 +2713,27 @@ impl Workbook { if matches!(lookup, Value::Error(_)) { return lookup; } - let Some((lookup_node, lookup_rows, lookup_columns)) = range_parts(&arguments[1]) - else { - return Value::Error(CalcError::InvalidArguments); + let lookup_input = match ArrayInput::new(&arguments[1], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; - let Some((return_node, return_rows, return_columns)) = range_parts(&arguments[2]) - else { - return Value::Error(CalcError::InvalidArguments); + let return_input = match ArrayInput::new(&arguments[2], self) { + Ok(input) => input, + Err(error) => return Value::Error(error), }; - let length = self.range_len(lookup_node); + let (lookup_rows, lookup_columns) = lookup_input.shape(); + let (return_rows, return_columns) = return_input.shape(); + let length = lookup_rows * lookup_columns; if (lookup_rows != 1 && lookup_columns != 1) || (return_rows != 1 && return_columns != 1) - || length != self.range_len(return_node) + || length != return_rows * return_columns { return Value::Error(CalcError::InvalidArguments); } for index in 0..length { - let candidate = self.range_value(lookup_node, index); + let candidate = lookup_input.value(self, index); if lookup_equal(&lookup, &candidate) { - return self.range_value(return_node, index); + return return_input.value(self, index); } } if arguments.len() == 4 { @@ -2755,12 +2860,84 @@ fn typed_compare(left: &Value, right: &Value) -> Result, } +/// Lookup inputs borrow constants and retain the existing range fast path. +enum ArrayInput<'a> { + Range { + node: usize, + rows: usize, + columns: usize, + }, + Constant(&'a ArrayValue), + Selection(reference::ReferenceView), + Computed(ArrayValue), +} + +impl<'a> ArrayInput<'a> { + fn new(expression: &'a Expr, workbook: &Workbook) -> Result { + match expression { + Expr::RangeNode { + node, + rows, + columns, + } => Ok(Self::Range { + node: *node, + rows: *rows, + columns: *columns, + }), + Expr::Array(array) => Ok(Self::Constant(array)), + Expr::Function(Function::Transpose | Function::MMult, _) => { + workbook.evaluate_array(expression).map(Self::Computed) + } + Expr::Reference(_) | Expr::Function(Function::ReferenceSpan, _) => { + workbook.reference_view(expression).map(Self::Selection) + } + Expr::Function(Function::Index, _) => match workbook.reference_view(expression) { + Ok(reference) => Ok(Self::Selection(reference)), + Err(CalcError::InvalidArguments) => { + workbook.evaluate_array(expression).map(Self::Computed) + } + Err(error) => Err(error), + }, + Expr::Error(error) => Err(error.clone()), + _ => Err(CalcError::InvalidArguments), + } + } + + fn shape(&self) -> (usize, usize) { + match self { + Self::Range { rows, columns, .. } => (*rows, *columns), + Self::Constant(array) => array.shape(), + Self::Selection(reference) => (reference.rows, reference.columns), + Self::Computed(array) => array.shape(), + } + } + + fn value(&self, workbook: &Workbook, index: usize) -> Value { + match self { + Self::Range { node, .. } => workbook.range_value(*node, index), + Self::Constant(array) => array.values[index].clone(), + Self::Selection(reference) => reference.value(workbook, index), + Self::Computed(array) => array.values[index].clone(), + } + } + + fn values(&self, workbook: &Workbook) -> Vec { + match self { + Self::Range { node, .. } => workbook.range_values(*node), + Self::Constant(array) => array.values.clone(), + Self::Selection(reference) => reference.array(workbook).values, + Self::Computed(array) => array.values.clone(), + } + } +} + impl ArrayValue { fn scalar(value: Value) -> Self { Self { @@ -2802,7 +2979,12 @@ fn broadcast_shape(shapes: &[(usize, usize)]) -> Result<(usize, usize), CalcErro /// lookups, criteria functions) produce one value and stop the search. fn contains_array_operand(expression: &Expr) -> bool { match expression { - Expr::RangeNode { .. } => true, + Expr::RangeNode { .. } + | Expr::Array(_) + | Expr::Function( + Function::Index | Function::ReferenceSpan | Function::Transpose | Function::MMult, + _, + ) => true, Expr::UnaryMinus(inner) | Expr::Percent(inner) => contains_array_operand(inner), Expr::Binary(_, left, right) => { contains_array_operand(left) || contains_array_operand(right) @@ -2859,7 +3041,10 @@ fn is_elementwise(function: Function) -> bool { | Function::YearFrac | Function::Days360 | Function::Pmt + | Function::Pv | Function::NormDist + | Function::NormSDist + | Function::NormSDistLegacy | Function::IsBlank | Function::IsNumber | Function::IsText @@ -2885,17 +3070,6 @@ fn literal(value: Value) -> Expr { } } -fn range_parts(expression: &Expr) -> Option<(usize, usize, usize)> { - match expression { - Expr::RangeNode { - node, - rows, - columns, - } => Some((*node, *rows, *columns)), - _ => None, - } -} - fn positive_index(value: Value) -> Result { let value = number(value)?; if value >= 1.0 && value.fract() == 0.0 && value <= usize::MAX as f64 { @@ -3245,6 +3419,94 @@ fn payment( Ok(-(present * growth + future) * rate / (timing * (growth - 1.0))) } +fn present_value( + rate: f64, + periods: f64, + payment: f64, + future: f64, + at_start: bool, +) -> Result { + if ![rate, periods, payment, future] + .iter() + .all(|value| value.is_finite()) + { + return Err(CalcError::InvalidNumber); + } + let result = if rate == 0.0 { + -payment * periods - future + } else { + let growth = (1.0 + rate).powf(periods); + if growth == 0.0 { + return Err(CalcError::DivisionByZero); + } + let growth_minus_one = if rate > -1.0 { + (periods * rate.ln_1p()).exp_m1() + } else { + growth - 1.0 + }; + -(future + payment * if at_start { 1.0 + rate } else { 1.0 } * growth_minus_one / rate) + / growth + }; + if result.is_finite() { + Ok(result) + } else { + Err(CalcError::InvalidNumber) + } +} + +/// Excel's periodic IRR iteration, keeping text/blanks out of the period count. +fn periodic_internal_rate_of_return(values: &[f64], guess: f64) -> Result { + if !guess.is_finite() + || guess <= -1.0 + || !values.iter().any(|value| *value > 0.0) + || !values.iter().any(|value| *value < 0.0) + { + return Err(CalcError::InvalidNumber); + } + let mut rate = guess; + for _ in 0..20 { + let base = 1.0 + rate; + let mut value = 0.0; + let mut slope = 0.0; + for (period, cash) in values.iter().enumerate() { + let discount = base.powi(period as i32); + value += cash / discount; + slope -= period as f64 * cash / (discount * base); + } + if slope == 0.0 || !slope.is_finite() { + return Err(CalcError::InvalidNumber); + } + let next = rate - value / slope; + if !next.is_finite() || next <= -1.0 { + return Err(CalcError::InvalidNumber); + } + if (next - rate).abs() < 1.0e-9 { + return Ok(next); + } + rate = next; + } + Err(CalcError::InvalidNumber) +} + +fn covariance(pairs: &[(f64, f64)], sample: bool) -> Result { + let correction = usize::from(sample); + if pairs.len() <= correction { + return Err(CalcError::DivisionByZero); + } + let mean_x = pairs.iter().map(|(x, _)| x).sum::() / pairs.len() as f64; + let mean_y = pairs.iter().map(|(_, y)| y).sum::() / pairs.len() as f64; + let value = pairs + .iter() + .map(|(x, y)| (x - mean_x) * (y - mean_y)) + .sum::() + / (pairs.len() - correction) as f64; + if value.is_finite() { + Ok(value) + } else { + Err(CalcError::InvalidNumber) + } +} + /// `NPV`: cash flows discounted from the end of the first period. fn net_present_value(rate: f64, values: &[f64]) -> Result { if rate == -1.0 { @@ -3800,6 +4062,7 @@ fn compile_expression( Expr::Text(value) => Expr::Text(value), Expr::Error(error) => Expr::Error(error), Expr::Empty => Expr::Empty, + Expr::Array(array) => Expr::Array(array), Expr::Reference(cell) => Expr::Reference(indices[&cell]), Expr::UnaryMinus(inner) => { Expr::UnaryMinus(Box::new(compile_expression(*inner, indices, range_nodes))) @@ -3839,6 +4102,7 @@ fn count_nodes(expression: &Expr) -> usize { Expr::Binary(_, left, right) => count_nodes(left) + count_nodes(right), Expr::Function(_, arguments) => arguments.iter().map(count_nodes).sum(), Expr::Range { members, .. } => members.as_ref().map_or(0, Vec::len), + Expr::Array(array) => array.values.len(), _ => 0, } } @@ -3875,6 +4139,7 @@ fn collect_dependencies( | Expr::Boolean(_) | Expr::Text(_) | Expr::Error(_) + | Expr::Array(_) | Expr::Empty => {} } } @@ -4066,6 +4331,29 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { fn parse_postfix(&mut self) -> Result { let mut expression = self.parse_primary()?; + loop { + self.skip_space(); + if self.peek() != Some(b':') { + break; + } + self.offset += 1; + // Only a bare A1 endpoint inherits the left sheet qualifier. + // Function arguments retain the formula's origin sheet. + self.skip_space(); + let right_start = self.offset; + let mut right = self.parse_primary()?; + let spelling = self.source[right_start..self.offset].trim(); + let inherited_sheet = match &expression { + Expr::Reference(cell) | Expr::Range { anchor: cell, .. } => cell.sheet, + _ => self.sheet, + }; + if matches!(right, Expr::Reference(_)) + && let Ok(cell) = parse_a1(spelling, inherited_sheet) + { + right = Expr::Reference(cell); + } + expression = join_reference_range(expression, right)?; + } loop { self.skip_space(); if self.peek() != Some(b'%') { @@ -4088,6 +4376,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { Ok(expression) } Some(b'"') => self.parse_string(), + Some(b'{') => self.parse_array_constant(), Some(b'\'') => self.parse_quoted_sheet_reference(), Some(b'#') => self.parse_error_literal(), Some(b'[') @@ -4110,6 +4399,80 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { self.remaining().chars().take(64).collect() } + fn parse_array_constant(&mut self) -> Result { + self.expect(b'{')?; + let mut values = Vec::new(); + let mut rows = 0; + let mut columns = None; + let mut row_columns = 0; + loop { + self.skip_space(); + let start = self.offset; + let value = match self.peek() { + Some(b'"') => self.parse_string()?, + Some(b'#') => self.parse_error_literal()?, + Some(b'+') | Some(b'-') => { + let negative = self.peek() == Some(b'-'); + self.offset += 1; + self.skip_space(); + let Expr::Number(number) = self.parse_number()? else { + unreachable!("number parser returns a number") + }; + Expr::Number(if negative { -number } else { number }) + } + Some(byte) if byte.is_ascii_digit() || byte == b'.' => self.parse_number()?, + Some(byte) if byte.is_ascii_alphabetic() => { + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphabetic()) { + self.offset += 1; + } + match self.source[start..self.offset] + .to_ascii_uppercase() + .as_str() + { + "TRUE" => Expr::Boolean(true), + "FALSE" => Expr::Boolean(false), + _ => return Err(FormulaError::UnexpectedToken(start)), + } + } + _ => return Err(FormulaError::UnexpectedToken(start)), + }; + let value = match value { + Expr::Number(value) if value.is_finite() => Value::Number(value), + Expr::Boolean(value) => Value::Boolean(value), + Expr::Text(value) => Value::Text(value), + Expr::Error(value) => Value::Error(value), + _ => return Err(FormulaError::UnexpectedToken(start)), + }; + if values.len() == MAX_RANGE_CELLS { + return Err(FormulaError::RangeTooLarge); + } + values.push(value); + row_columns += 1; + self.skip_space(); + match self.peek() { + Some(b',') => self.offset += 1, + Some(b';') | Some(b'}') => { + if columns.is_some_and(|width| width != row_columns) { + return Err(FormulaError::UnexpectedToken(self.offset)); + } + columns = Some(row_columns); + rows += 1; + row_columns = 0; + let finished = self.peek() == Some(b'}'); + self.offset += 1; + if finished { + return Ok(Expr::Array(ArrayValue { + rows, + columns: columns.expect("completed row"), + values, + })); + } + } + _ => return Err(FormulaError::UnexpectedToken(self.offset)), + } + } + } + fn parse_error_literal(&mut self) -> Result { let start = self.offset; while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || matches!(byte, b'#' | b'/' | b'!' | b'?')) @@ -4196,7 +4559,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { return Ok(Expr::Boolean(false)); } match parse_a1(token, self.sheet) { - Ok(first) => self.parse_range_tail(first, self.sheet), + Ok(first) => Ok(Expr::Reference(first)), Err(_) => self.parse_defined_name(token), } } @@ -4205,8 +4568,16 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { /// same sheet table and a bounded depth for names that use names. A name /// scoped to the formula's sheet wins over a workbook-level one. fn parse_defined_name(&mut self, token: &str) -> Result { + self.parse_defined_name_in_sheet(token, self.sheet) + } + + fn parse_defined_name_in_sheet( + &mut self, + token: &str, + sheet: u32, + ) -> Result { let name = token.trim_start_matches('$'); - let Some(definition) = self.defined_names.resolve(self.sheet, &name.to_lowercase()) else { + let Some(definition) = self.defined_names.resolve(sheet, &name.to_lowercase()) else { return Err(FormulaError::UnknownName(name.into())); }; if self.name_depth >= MAX_NAME_DEPTH { @@ -4214,7 +4585,7 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { } let mut inner = Parser::new_structured( definition, - self.sheet, + sheet, self.sheet_names, self.defined_names, self.structured, @@ -4305,26 +4676,23 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { return Ok(Expr::Error(CalcError::InvalidReference)); } let start = self.offset; - while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'$') { + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || matches!(byte, b'$' | b'_' | b'.')) + { self.offset += 1; } - let first = parse_a1(&self.source[start..self.offset], sheet)?; - self.parse_range_tail(first, sheet) - } - - fn parse_range_tail(&mut self, first: CellId, sheet: u32) -> Result { - self.skip_space(); - if self.peek() != Some(b':') { - return Ok(Expr::Reference(first)); - } - self.offset += 1; - self.skip_space(); - let second_start = self.offset; - while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'$') { - self.offset += 1; + let token = &self.source[start..self.offset]; + match parse_a1(token, sheet) { + Ok(first) => Ok(Expr::Reference(first)), + Err(_) + if self + .defined_names + .resolve(sheet, &token.trim_start_matches('$').to_lowercase()) + .is_some() => + { + self.parse_defined_name_in_sheet(token, sheet) + } + Err(error) => Err(error), } - let second = parse_a1(&self.source[second_start..self.offset], sheet)?; - expand_range(first, second) } fn resolve_sheet(&self, name: &str) -> Result { @@ -4393,6 +4761,12 @@ impl<'source, 'sheets> Parser<'source, 'sheets> { /// [`supported_function_names`] both read it, and a test keeps /// `docs/FUNCTIONS.md` in step with it so documented counts cannot drift. const FUNCTION_REGISTRY: &[(&str, Function)] = &[ + ("TRANSPOSE", Function::Transpose), + ("MMULT", Function::MMult), + ("DAVERAGE", Function::DAverage), + ("DMAX", Function::DMax), + ("DMIN", Function::DMin), + ("DSTDEV", Function::DStDev), ("SUM", Function::Sum), ("AVERAGE", Function::Average), ("MIN", Function::Min), @@ -4457,10 +4831,18 @@ const FUNCTION_REGISTRY: &[(&str, Function)] = &[ ("WORKDAY", Function::WorkDay), ("LOOKUP", Function::Lookup), ("PMT", Function::Pmt), + ("PV", Function::Pv), + ("IRR", Function::Irr), ("NPV", Function::Npv), ("XNPV", Function::Xnpv), ("XIRR", Function::Xirr), ("NORMDIST", Function::NormDist), + ("NORM.DIST", Function::NormDist), + ("NORMSDIST", Function::NormSDistLegacy), + ("NORM.S.DIST", Function::NormSDist), + ("COVAR", Function::CovarianceP), + ("COVARIANCE.P", Function::CovarianceP), + ("COVARIANCE.S", Function::CovarianceS), ("AVERAGEA", Function::AverageA), ("CORREL", Function::Correl), ("ISBLANK", Function::IsBlank), @@ -4498,9 +4880,11 @@ pub fn supported_function_names() -> impl Iterator { fn parse_function_name(name: &str) -> Result { let upper = name.to_ascii_uppercase(); + let registered = upper.strip_prefix("_XLFN.").unwrap_or(&upper); + let registered = registered.strip_prefix("_XLWS.").unwrap_or(registered); FUNCTION_REGISTRY .iter() - .find(|(candidate, _)| *candidate == upper) + .find(|(candidate, _)| *candidate == registered) .map(|(_, function)| *function) .ok_or(FormulaError::UnsupportedFunction(upper)) } @@ -4662,10 +5046,286 @@ fn expand_range(first: CellId, second: CellId) -> Result { }) } +fn reference_bounds(expression: &Expr) -> Option<(CellId, CellId)> { + match expression { + Expr::Reference(cell) => Some((*cell, *cell)), + Expr::Range { + anchor, + rows, + columns, + members: None, + } => Some(( + *anchor, + CellId::new( + anchor.sheet, + anchor.row + *rows as u32 - 1, + anchor.column + *columns as u32 - 1, + ), + )), + Expr::Function(Function::Index, arguments) => arguments.first().and_then(reference_bounds), + Expr::Function(Function::ReferenceSpan, arguments) => { + arguments.get(2).and_then(reference_bounds) + } + _ => None, + } +} + +fn join_reference_range(left: Expr, right: Expr) -> Result { + let left_bounds = reference_bounds(&left); + let right_bounds = reference_bounds(&right); + let left_deleted = matches!(left, Expr::Error(CalcError::InvalidReference)); + let right_deleted = matches!(right, Expr::Error(CalcError::InvalidReference)); + if (left_deleted && (right_deleted || right_bounds.is_some())) + || (right_deleted && left_bounds.is_some()) + { + return Ok(Expr::Error(CalcError::InvalidReference)); + } + let (Some((first, first_end)), Some((second, second_end))) = (left_bounds, right_bounds) else { + let kind = if matches!(left, Expr::Number(_)) || matches!(right, Expr::Number(_)) { + "range endpoint is number" + } else if matches!(left, Expr::Function(_, _)) || matches!(right, Expr::Function(_, _)) { + "range endpoint is function" + } else { + "range endpoints must be references" + }; + return Err(FormulaError::InvalidReference(kind.into())); + }; + if first.sheet != second.sheet { + return Err(FormulaError::InvalidReference( + "range endpoints cross sheets".into(), + )); + } + let envelope = expand_range( + CellId::new( + first.sheet, + first.row.min(second.row), + first.column.min(second.column), + ), + CellId::new( + first.sheet, + first_end.row.max(second_end.row), + first_end.column.max(second_end.column), + ), + )?; + if matches!(left, Expr::Function(_, _)) || matches!(right, Expr::Function(_, _)) { + Ok(Expr::Function( + Function::ReferenceSpan, + vec![left, right, envelope], + )) + } else { + Ok(envelope) + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn deleted_and_qualified_range_endpoints_follow_reference_semantics() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Input"); + workbook.define_sheet(1, "Other sheet"); + workbook.set_number(CellId::new(1, 0, 0), 3.0); + workbook.set_number(CellId::new(1, 1, 0), 7.0); + for formula in [ + "=SUM('Other sheet'!A1:A2)", + "=SUM('Other sheet'!A1:'Other sheet'!A2)", + "=SUM(('Other sheet'!A1):('Other sheet'!A2))", + ] { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(10.0), "{formula}"); + } + let changed = workbook.set_number(CellId::new(1, 1, 0), 9.0); + assert!(changed.evaluated.contains(&cell(0, 3))); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(12.0)); + for formula in [ + "=SUM(#REF!:#REF!)", + "=SUM(A1:#REF!)", + "=SUM(#REF!:A1)", + "=SUM('Other sheet'!#REF!:A2)", + "=SUM('Other sheet'!A1:'Other sheet'!#REF!)", + ] { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!( + workbook.value(cell(0, 3)), + Value::Error(CalcError::InvalidReference), + "{formula}" + ); + } + workbook + .set_formula(cell(0, 3), "=IFERROR(SUM(#REF!:#REF!),17)") + .unwrap(); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(17.0)); + for formula in [ + "=SUM(Input!A1:'Other sheet'!A2)", + "=SUM(A1:2)", + "=SUM(1:A2)", + "=SUM(A1:[1]Sheet!B2)", + ] { + assert!( + workbook.set_formula(cell(0, 3), formula).is_err(), + "{formula}" + ); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(17.0)); + } + } + + #[test] + fn added_financial_and_statistical_functions_match_reference_values() { + let mut workbook = Workbook::default(); + let examples = [ + ("=PV(0,10,-100)", 1000.0, 1e-10), + ("=PV(0,0,0,100)", -100.0, 1e-10), + ("=PV(0.1,1,-110)", 100.0, 1e-10), + ("=PV(0.1,1,-100,,1)", 100.0, 1e-10), + ("=PV(0.08/12,240,500,,0)", -59777.15, 0.005), + ("=PV(1e-12,1,-100)", 100.0 / (1.0 + 1e-12), 1e-10), + ("=IRR({-100,110})", 0.1, 1e-9), + ("=IRR({-100,\"ignore\",FALSE,0,121})", 0.1, 1e-9), + ("=IRR({-100,90})", -0.1, 1e-9), + ( + "=IRR({-70000,12000,15000,18000,21000,26000})", + 0.086630948036531, + 1e-9, + ), + ("=COVAR({3,2,4,5,6},{9,7,12,15,17})", 5.2, 1e-10), + ("=COVARIANCE.P({0,2},{2,0})", -1.0, 1e-10), + ("=COVARIANCE.S({0,2},{2,0})", -2.0, 1e-10), + ("=COVAR({1,\"ignored\",3},{2,999,6})", 2.0, 1e-10), + ("=NORMSDIST(0)", 0.5, 1e-10), + ("=_xlfn.NORM.S.DIST(1.333333,TRUE)", 0.908788726, 1e-9), + ("=NORM.S.DIST(1.333333,FALSE)", 0.164010148, 1e-9), + ( + "=NORM.DIST(0,0,1,FALSE)", + 1.0 / (2.0 * std::f64::consts::PI).sqrt(), + 1e-10, + ), + ]; + for (formula, expected, tolerance) in examples { + workbook.set_formula(cell(0, 0), formula).unwrap(); + let Value::Number(value) = workbook.value(cell(0, 0)) else { + panic!("{formula} did not return a number") + }; + assert!( + (value - expected).abs() < tolerance, + "{formula}: {value} != {expected}" + ); + } + for (formula, error) in [ + ("=IRR({1,2})", CalcError::InvalidNumber), + ("=IRR({-1,1},-1)", CalcError::InvalidNumber), + ("=IRR({-1,#N/A,2})", CalcError::NotAvailable), + ("=PV(-1,1,1)", CalcError::DivisionByZero), + ("=PV(0,1,#REF!)", CalcError::InvalidReference), + ("=COVARIANCE.S({1},{2})", CalcError::DivisionByZero), + ("=COVAR({1,2},{3})", CalcError::NotAvailable), + ("=COVAR({1,#N/A},{3,4})", CalcError::NotAvailable), + ("=NORMSDIST(0,FALSE)", CalcError::InvalidArguments), + ("=NORM.S.DIST(0)", CalcError::InvalidArguments), + ] { + workbook.set_formula(cell(0, 0), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 0)), Value::Error(error), "{formula}"); + } + } + + #[test] + fn array_constants_feed_aggregates_broadcasting_and_lookups() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 10.0); + workbook.set_number(cell(1, 0), 20.0); + let examples = [ + ("=SUM({1,2;3,4})", Value::Number(10.0)), + ("=SUMPRODUCT(A1:A2,{2;3})", Value::Number(80.0)), + ("=SUM(IF({TRUE,FALSE},A1,2))", Value::Number(12.0)), + ("=SUM({1,2}+{10;20})", Value::Number(66.0)), + ("=INDEX({1,2;3,4},2,1)", Value::Number(3.0)), + ("=INDEX({1,2,3},2)", Value::Number(2.0)), + ("=MATCH(20,{10;20;30},0)", Value::Number(2.0)), + ("=VLOOKUP(2,{1,10;2,20},2,FALSE)", Value::Number(20.0)), + ("=HLOOKUP(2,{1,2;10,20},2,FALSE)", Value::Number(20.0)), + ("=XLOOKUP(2,{1,2},{10,20})", Value::Number(20.0)), + ("=LOOKUP(2,{1,2,3},{10,20,30})", Value::Number(20.0)), + ("=SUM({-2,+3,1e2})", Value::Number(101.0)), + ("=SUM({1,#N/A})", Value::Error(CalcError::NotAvailable)), + ( + "=TEXTJOIN(\"/\",TRUE,{\"a\",\"\";\"b\",\"c\"})", + Value::Text("a/b/c".into()), + ), + ("=SUM(_xlfn.IFERROR({1,#N/A},0))", Value::Number(1.0)), + ]; + for (formula, expected) in examples { + workbook.set_formula(cell(0, 3), formula).unwrap(); + assert_eq!(workbook.value(cell(0, 3)), expected, "{formula}"); + } + workbook.define_name("weights", "{2;3}"); + workbook + .set_formula(cell(0, 3), "=SUMPRODUCT(A1:A2,weights)") + .unwrap(); + let changed = workbook.set_number(cell(1, 0), 30.0); + assert!(changed.evaluated.contains(&cell(0, 3))); + assert_eq!(workbook.value(cell(0, 3)), Value::Number(110.0)); + } + + #[test] + fn malformed_array_constants_are_rejected_atomically() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 17.0); + for formula in [ + "={}", + "={1,}", + "={;1}", + "={1;}", + "={1,2;3}", + "={1,A1}", + "={SUM(1)}", + "={{1}}", + "={1+2}", + "={1%}", + "={1e999}", + "={TRUEFALSE}", + "={1;2,3}", + ] { + assert!( + workbook.set_formula(cell(0, 0), formula).is_err(), + "{formula}" + ); + assert_eq!(workbook.value(cell(0, 0)), Value::Number(17.0)); + } + let oversized = format!("={{{}0}}", "0,".repeat(MAX_RANGE_CELLS)); + assert_eq!( + workbook.set_formula(cell(0, 0), &oversized), + Err(FormulaError::RangeTooLarge) + ); + assert_eq!(workbook.value(cell(0, 0)), Value::Number(17.0)); + } + + #[test] + fn qualified_names_bind_to_the_named_sheet_and_recalculate() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Input"); + workbook.define_sheet(1, "Other sheet"); + workbook.define_sheet_name(0, "rate.total", "A1*2"); + workbook.define_sheet_name(1, "rate.total", "A1*3"); + workbook.set_number(CellId::new(0, 0, 0), 10.0); + workbook.set_number(CellId::new(1, 0, 0), 20.0); + workbook + .set_formula(cell(0, 1), "=Input!rate.total+'Other sheet'!rate.total") + .unwrap(); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(80.0)); + let changed = workbook.set_number(CellId::new(1, 0, 0), 30.0); + assert!(changed.evaluated.contains(&cell(0, 1))); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(110.0)); + assert!(matches!( + workbook.set_formula(cell(0, 1), "=_xlfn.MISSING(1)"), + Err(FormulaError::UnsupportedFunction(_)) + )); + assert!(matches!( + workbook.set_formula(cell(0, 1), "='[1]Other sheet'!rate.total"), + Err(FormulaError::ExternalReference(_)) + )); + } + fn cell(row: u32, column: u32) -> CellId { CellId::new(0, row, column) } diff --git a/crates/omasheets-calc/src/matrix.rs b/crates/omasheets-calc/src/matrix.rs new file mode 100644 index 0000000..96b29c0 --- /dev/null +++ b/crates/omasheets-calc/src/matrix.rs @@ -0,0 +1,117 @@ +//! Bounded matrix operations with explicit shape and numeric-input checks. +use super::*; + +impl Workbook { + pub(super) fn matrix_array( + &self, + function: Function, + arguments: &[Expr], + ) -> Result { + if function == Function::Transpose { + let [input] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let input = self.evaluate_array(input)?; + let mut values = Vec::with_capacity(input.values.len()); + for column in 0..input.columns { + for row in 0..input.rows { + values.push(input.values[row * input.columns + column].clone()); + } + } + return Ok(ArrayValue { + rows: input.columns, + columns: input.rows, + values, + }); + } + let [left, right] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let left = self.evaluate_array(left)?; + let right = self.evaluate_array(right)?; + if left.columns != right.rows { + return Err(CalcError::InvalidValue); + } + let count = left + .rows + .checked_mul(right.columns) + .filter(|count| *count <= MAX_RANGE_CELLS) + .ok_or(CalcError::InvalidNumber)?; + if count + .checked_mul(left.columns) + .is_none_or(|terms| terms > 50_000_000) + { + return Err(CalcError::InvalidNumber); + } + let numbers = |array: ArrayValue| { + array + .values + .into_iter() + .map(|value| match value { + Value::Number(value) if value.is_finite() => Ok(value), + Value::Error(error) => Err(error), + _ => Err(CalcError::InvalidValue), + }) + .collect::, _>>() + }; + let (rows, inner, columns) = (left.rows, left.columns, right.columns); + let left = numbers(left)?; + let right = numbers(right)?; + let mut values = Vec::with_capacity(count); + for row in 0..rows { + for column in 0..columns { + let mut sum = 0.0; + for k in 0..inner { + sum += left[row * inner + k] * right[k * columns + column]; + } + values.push(if sum.is_finite() { + Value::Number(sum) + } else { + Value::Error(CalcError::InvalidNumber) + }); + } + } + Ok(ArrayValue { + rows, + columns, + values, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn matrix_composition_preserves_dimensions_types_and_dependencies() { + let mut workbook = Workbook::default(); + for (formula, expected) in [ + ("=SUM(MMULT({1,2;3,4},{5,6;7,8}))", 134.0), + ("=INDEX(TRANSPOSE({1,2,3;4,5,6}),3,2)", 6.0), + ("=MMULT({1,2,3},TRANSPOSE({4,5,6}))", 32.0), + ("=SUM(TRANSPOSE({1,2;3,4})*{1,2;3,4})", 29.0), + ("=IFERROR(MMULT({1,2},{3,4}),17)", 17.0), + ("=IFERROR(MMULT({1,TRUE},{3;4}),18)", 18.0), + ("=IFERROR(MMULT({1,\"2\"},{3;4}),19)", 19.0), + ("=IFERROR(MMULT({1,#N/A},{3;4}),20)", 20.0), + ("=INDEX(TRANSPOSE({1,\"text\";TRUE,4}),2,2)", 4.0), + ] { + let cell = CellId::new(0, 9, 9); + workbook.set_formula(cell, formula).unwrap(); + assert_eq!(workbook.value(cell), Value::Number(expected), "{formula}"); + } + workbook.set_number(CellId::new(0, 0, 0), 2.0); + workbook.set_number(CellId::new(0, 1, 0), 3.0); + workbook + .set_formula(CellId::new(0, 0, 2), "=MMULT(TRANSPOSE(A1:A2),A1:A2)") + .unwrap(); + assert_eq!(workbook.value(CellId::new(0, 0, 2)), Value::Number(13.0)); + workbook.set_number(CellId::new(0, 1, 0), 4.0); + assert_eq!(workbook.value(CellId::new(0, 0, 2)), Value::Number(20.0)); + workbook.clear(CellId::new(0, 1, 0)); + assert_eq!( + workbook.value(CellId::new(0, 0, 2)), + Value::Error(CalcError::InvalidValue) + ); + } +} diff --git a/crates/omasheets-calc/src/reference.rs b/crates/omasheets-calc/src/reference.rs new file mode 100644 index 0000000..3b5601f --- /dev/null +++ b/crates/omasheets-calc/src/reference.rs @@ -0,0 +1,516 @@ +//! Reference-valued selections retain a bounded, stable dependency envelope. +use super::*; + +/// Narrow only provably constant axes, after stable bindings have been applied. +/// The persisted parsed shape remains unchanged, including legacy INDEX bindings. +pub(super) fn narrow_reference_dependencies(expression: Expr) -> Expr { + match expression { + Expr::UnaryMinus(inner) => { + Expr::UnaryMinus(Box::new(narrow_reference_dependencies(*inner))) + } + Expr::Percent(inner) => Expr::Percent(Box::new(narrow_reference_dependencies(*inner))), + Expr::Binary(op, left, right) => Expr::Binary( + op, + Box::new(narrow_reference_dependencies(*left)), + Box::new(narrow_reference_dependencies(*right)), + ), + Expr::Function(function, arguments) => { + let mut arguments: Vec<_> = arguments + .into_iter() + .map(narrow_reference_dependencies) + .collect(); + if function == Function::Index && matches!(arguments.len(), 2 | 3) { + narrow_index(&mut arguments); + } else if function == Function::ReferenceSpan + && arguments.len() == 3 + && matches!(arguments[2], Expr::Range { members: None, .. }) + && let (Some((a, b)), Some((c, d))) = ( + reference_bounds(&arguments[0]), + reference_bounds(&arguments[1]), + ) + && a.sheet == c.sheet + && let Ok(envelope) = expand_range( + CellId::new(a.sheet, a.row.min(c.row), a.column.min(c.column)), + CellId::new(a.sheet, b.row.max(d.row), b.column.max(d.column)), + ) + { + arguments[2] = envelope; + } + Expr::Function(function, arguments) + } + other => other, + } +} + +fn narrow_index(arguments: &mut Vec) { + let Expr::Range { + anchor, + members, + rows, + columns, + } = &arguments[0] + else { + return; + }; + let (anchor, members, rows, columns) = (*anchor, members.clone(), *rows, *columns); + let literal = |expression: &Expr, bound: usize| match expression { + Expr::Number(value) + if value.is_finite() && *value >= 1.0 && value.trunc() <= bound as f64 => + { + Some(value.trunc() as usize - 1) + } + _ => None, + }; + let horizontal = rows == 1 && arguments.len() == 2; + let selected_row = if horizontal { + None + } else { + literal(&arguments[1], rows) + }; + let selected_column = if horizontal { + literal(&arguments[1], columns) + } else { + arguments.get(2).and_then(|arg| literal(arg, columns)) + }; + if selected_row.is_none() && selected_column.is_none() { + return; + } + // Make the original omitted-column semantics explicit before changing shape. + if arguments.len() == 2 && !horizontal { + arguments.push(Expr::Number(if columns == 1 { 1.0 } else { 0.0 })); + } + let first_row = selected_row.unwrap_or(0); + let first_column = selected_column.unwrap_or(0); + let new_rows = if selected_row.is_some() { 1 } else { rows }; + let new_columns = if selected_column.is_some() { + 1 + } else { + columns + }; + let (new_anchor, new_members) = if let Some(members) = members { + let selected: Vec<_> = (first_row..first_row + new_rows) + .flat_map(|row| { + let members = &members; + (first_column..first_column + new_columns) + .map(move |column| members[row * columns + column]) + }) + .collect(); + (selected[0], Some(selected)) + } else { + ( + CellId::new( + anchor.sheet, + anchor.row + first_row as u32, + anchor.column + first_column as u32, + ), + None, + ) + }; + arguments[0] = Expr::Range { + anchor: new_anchor, + members: new_members, + rows: new_rows, + columns: new_columns, + }; + if selected_row.is_some() || (horizontal && selected_column.is_some()) { + arguments[1] = Expr::Number(1.0); + } + if !horizontal && selected_column.is_some() { + arguments[2] = Expr::Number(1.0); + } +} + +#[derive(Clone, Copy)] +pub(super) struct ReferenceView { + node: Option, + anchor: CellId, + row: usize, + column: usize, + pub(super) rows: usize, + pub(super) columns: usize, +} + +impl ReferenceView { + fn cell(self, workbook: &Workbook, row: usize, column: usize) -> CellId { + if let Some(node) = self.node { + match workbook.range_shape(node) { + RangeShape::Rectangle { anchor, .. } => CellId::new( + anchor.sheet, + anchor.row + (self.row + row) as u32, + anchor.column + (self.column + column) as u32, + ), + RangeShape::Members { columns, .. } => { + let index = (self.row + row) * columns + self.column + column; + workbook.cells[workbook.cells[node].dependencies[index]].id + } + } + } else { + CellId::new( + self.anchor.sheet, + self.anchor.row + (self.row + row) as u32, + self.anchor.column + (self.column + column) as u32, + ) + } + } + + fn position(self, workbook: &Workbook, cell: CellId) -> Option<(usize, usize)> { + let (row, column) = match self.node.map(|node| (node, workbook.range_shape(node))) { + Some((node, RangeShape::Members { columns, .. })) => { + let index = workbook.cells[node] + .dependencies + .iter() + .position(|index| workbook.cells[*index].id == cell)?; + (index / columns, index % columns) + } + _ => { + if cell.sheet != self.anchor.sheet { + return None; + } + ( + cell.row.checked_sub(self.anchor.row)? as usize, + cell.column.checked_sub(self.anchor.column)? as usize, + ) + } + }; + let row = row.checked_sub(self.row)?; + let column = column.checked_sub(self.column)?; + (row < self.rows && column < self.columns).then_some((row, column)) + } + + fn select(self, row: usize, column: usize, rows: usize, columns: usize) -> Self { + Self { + row: self.row + row, + column: self.column + column, + rows, + columns, + ..self + } + } + + pub(super) fn value(self, workbook: &Workbook, index: usize) -> Value { + workbook.value(self.cell(workbook, index / self.columns, index % self.columns)) + } + + pub(super) fn array(self, workbook: &Workbook) -> ArrayValue { + ArrayValue { + rows: self.rows, + columns: self.columns, + values: (0..self.rows * self.columns) + .map(|index| self.value(workbook, index)) + .collect(), + } + } + + pub(super) fn scalar(self, workbook: &Workbook) -> Value { + if self.rows == 1 && self.columns == 1 { + return self.value(workbook, 0); + } + let origin = workbook.evaluating.get(); + for index in 0..self.rows * self.columns { + let cell = self.cell(workbook, index / self.columns, index % self.columns); + if (self.columns == 1 || cell.column == origin.column) + && (self.rows == 1 || cell.row == origin.row) + { + return workbook.value(cell); + } + } + Value::Error(CalcError::InvalidValue) + } + + pub(super) fn visit(self, workbook: &Workbook, mut visit: impl FnMut(&Value)) { + if self + .node + .is_none_or(|node| matches!(workbook.range_shape(node), RangeShape::Rectangle { .. })) + { + workbook.for_each_rectangle_cell( + self.cell(workbook, 0, 0), + self.rows, + self.columns, + |_, index| visit(&workbook.cells[index].value), + ); + } else { + for index in 0..self.rows * self.columns { + visit(&self.value(workbook, index)); + } + } + } +} + +impl Workbook { + pub(super) fn reference_view( + &self, + expression: &Expr, + ) -> Result { + match expression { + Expr::Reference(index) => Ok(ReferenceView { + node: None, + anchor: self.cells[*index].id, + row: 0, + column: 0, + rows: 1, + columns: 1, + }), + Expr::RangeNode { + node, + rows, + columns, + } => Ok(ReferenceView { + node: Some(*node), + anchor: self.cells[*node].id, + row: 0, + column: 0, + rows: *rows, + columns: *columns, + }), + Expr::Function(Function::Index, arguments) if matches!(arguments.len(), 2 | 3) => { + let input = self.reference_view(&arguments[0])?; + let (row, column, rows, columns) = + self.index_selection(arguments, input.rows, input.columns)?; + Ok(input.select(row, column, rows, columns)) + } + Expr::Function(Function::ReferenceSpan, arguments) => self.reference_span(arguments), + Expr::Error(error) => Err(error.clone()), + _ => Err(CalcError::InvalidArguments), + } + } + + pub(super) fn reference_span( + &self, + arguments: &[Expr], + ) -> Result { + let [first, last, envelope] = arguments else { + return Err(CalcError::InvalidArguments); + }; + let first = self.reference_view(first)?; + let last = self.reference_view(last)?; + let envelope = self.reference_view(envelope)?; + let endpoints = [ + first.cell(self, 0, 0), + first.cell(self, first.rows - 1, first.columns - 1), + last.cell(self, 0, 0), + last.cell(self, last.rows - 1, last.columns - 1), + ]; + let mut min_row = usize::MAX; + let mut min_column = usize::MAX; + let mut max_row = 0; + let mut max_column = 0; + for cell in endpoints { + let (row, column) = envelope + .position(self, cell) + .ok_or(CalcError::InvalidReference)?; + min_row = min_row.min(row); + min_column = min_column.min(column); + max_row = max_row.max(row); + max_column = max_column.max(column); + } + Ok(envelope.select( + min_row, + min_column, + max_row - min_row + 1, + max_column - min_column + 1, + )) + } + + fn index_selection( + &self, + arguments: &[Expr], + rows: usize, + columns: usize, + ) -> Result<(usize, usize, usize, usize), CalcError> { + if !matches!(arguments.len(), 2 | 3) { + return Err(CalcError::InvalidArguments); + } + let index = |expression: &Expr| -> Result { + let value = number(self.evaluate(expression))?; + if !value.is_finite() || value < 0.0 || value > usize::MAX as f64 { + return Err(CalcError::InvalidReference); + } + Ok(value.trunc() as usize) + }; + let mut row = index(&arguments[1])?; + let column = if let Some(column) = arguments.get(2) { + index(column)? + } else if rows == 1 { + let column = row; + row = 1; + column + } else if columns == 1 { + 1 + } else { + 0 + }; + if row > rows || column > columns { + return Err(CalcError::InvalidReference); + } + Ok(( + row.saturating_sub(1), + column.saturating_sub(1), + if row == 0 { rows } else { 1 }, + if column == 0 { columns } else { 1 }, + )) + } + + pub(super) fn index_array(&self, arguments: &[Expr]) -> Result { + let Some(first) = arguments.first() else { + return Err(CalcError::InvalidArguments); + }; + let input = ArrayInput::new(first, self)?; + let (input_rows, input_columns) = input.shape(); + let (row, column, rows, columns) = + self.index_selection(arguments, input_rows, input_columns)?; + let mut values = Vec::with_capacity(rows * columns); + for r in row..row + rows { + for c in column..column + columns { + values.push(input.value(self, r * input_columns + c)); + } + } + Ok(ArrayValue { + rows, + columns, + values, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cell(row: u32, column: u32) -> CellId { + CellId::new(0, row, column) + } + + #[test] + fn index_references_supply_dynamic_endpoints_and_whole_axes() { + let mut workbook = Workbook::default(); + for row in 0..3 { + workbook.set_number(cell(row, 0), (row + 1) as f64 * 10.0); + workbook.set_number(cell(row, 1), (row + 1) as f64); + } + workbook.set_number(cell(0, 3), 2.0); + for (source, expected) in [ + ("=SUM(A1:INDEX(A1:A3,D1))", 30.0), + ("=SUM(INDEX(A1:A3,2):INDEX(A1:A3,3))", 50.0), + ("=SUM(INDEX(A1:B3,0,2))", 6.0), + ("=SUM(INDEX(A1:B3,2,0))", 22.0), + ("=SUM(INDEX(A1:B3,0,0))", 66.0), + ("=SUM(INDEX(A1:B1,0))", 11.0), + ("=MATCH(20,INDEX(A1:B3,0,1),0)", 2.0), + ("=VLOOKUP(20,INDEX(A1:B3,0,0),2,FALSE)", 2.0), + ("=SUM(A1:INDEX(INDEX(A1:A3,0),D1))", 30.0), + ("=SUM(INDEX(A1:B3,0,1):INDEX(A1:B3,0,2))", 66.0), + ("=SUM(INDEX({1,2;3,4},0,2))", 6.0), + ("=SUM(INDEX({1,2;3,4},2,0))", 7.0), + ("=SUM(INDEX(A1:A3,0)*B1:B3)", 140.0), + ("=IFERROR(SUM(A1:INDEX(A1:A3,4)),17)", 17.0), + ] { + workbook.set_formula(cell(5, 6), source).unwrap(); + assert_eq!( + workbook.value(cell(5, 6)), + Value::Number(expected), + "{source}" + ); + } + workbook.set_formula(cell(1, 2), "=INDEX(A1:A3,0)").unwrap(); + assert_eq!(workbook.value(cell(1, 2)), Value::Number(20.0)); + workbook + .set_formula(cell(5, 6), "=SUM(A1:INDEX(A1:A3,D1))") + .unwrap(); + workbook.set_number(cell(0, 3), 3.0); + assert_eq!(workbook.value(cell(5, 6)), Value::Number(60.0)); + workbook.set_number(cell(2, 0), 40.0); + assert_eq!(workbook.value(cell(5, 6)), Value::Number(70.0)); + } + + #[test] + fn reference_endpoints_keep_function_selectors_on_the_origin_sheet() { + let mut workbook = Workbook::default(); + workbook.define_sheet(0, "Data"); + workbook.define_sheet(1, "Report"); + for row in 0..3 { + workbook.set_number(CellId::new(0, row, 0), (row + 1) as f64 * 10.0); + } + workbook.set_number(CellId::new(0, 0, 1), 99.0); + workbook.set_number(CellId::new(1, 0, 1), 2.0); + for formula in [ + "=SUM(Data!A1:INDEX(Data!A1:A3,B1))", + "=SUM(INDEX(Data!A1:A3,1):INDEX(Data!A1:A3,B1))", + "=SUM(Data!A1:(INDEX(Data!A1:A3,B1)))", + "=SUM(Data!A1:A2)", + ] { + let target = CellId::new(1, 0, 2); + workbook.set_formula(target, formula).unwrap(); + assert_eq!(workbook.value(target), Value::Number(30.0), "{formula}"); + } + } + + #[test] + fn constant_index_axes_do_not_create_false_cycles() { + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 10.0); + workbook.set_number(cell(1, 0), 20.0); + workbook.set_number(cell(2, 0), 30.0); + workbook.set_number(cell(0, 3), 2.0); + workbook + .set_formula(cell(0, 1), "=SUM(A1:INDEX(A1:B3,D1,1))") + .unwrap(); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(30.0)); + workbook.set_number(cell(0, 3), 3.0); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(60.0)); + workbook.set_number(cell(2, 0), 40.0); + assert_eq!(workbook.value(cell(0, 1)), Value::Number(70.0)); + for (formula, expected) in [ + ("=SUM(INDEX(A1:B3,2))", 20.0), + ("=INDEX(A1:B1,1)", 10.0), + ("=INDEX(A1:B3,2,1)", 20.0), + ("=SUM(INDEX(A1:B3,0,1))", 70.0), + ("=IFERROR(INDEX(A1:B3,4,1),99)", 99.0), + ] { + workbook.set_formula(cell(6, 6), formula).unwrap(); + assert_eq!( + workbook.value(cell(6, 6)), + Value::Number(expected), + "{formula}" + ); + } + assert!(matches!( + workbook.set_formula(cell(0, 0), "=INDEX(A1:B3,1,1)"), + Err(FormulaError::Cycle(_)) + )); + let parsed = ParsedFormula::parse("=INDEX(A1:B3,2,1)", 0, &HashMap::new()).unwrap(); + assert_eq!(parsed.reference_count(), 6); + let mapped = parsed.map_references(|id| CellId::new(id.sheet, 2 - id.row, id.column)); + workbook.set_parsed_formula(cell(6, 6), mapped).unwrap(); + assert_eq!(workbook.value(cell(6, 6)), Value::Number(20.0)); + } + + #[test] + fn reference_envelope_rebinds_and_remains_sparse() { + let parsed = ParsedFormula::parse("=SUM(A1:INDEX(A1:A3,2))", 0, &HashMap::new()).unwrap(); + let mapped = parsed.map_references(|id| CellId::new(id.sheet, 2 - id.row, id.column)); + let mut workbook = Workbook::default(); + workbook.set_number(cell(0, 0), 30.0); + workbook.set_number(cell(1, 0), 20.0); + workbook.set_number(cell(2, 0), 10.0); + workbook.set_parsed_formula(cell(5, 1), mapped).unwrap(); + assert_eq!(workbook.value(cell(5, 1)), Value::Number(30.0)); + workbook.set_number(cell(2, 0), 50.0); + assert_eq!(workbook.value(cell(5, 1)), Value::Number(70.0)); + + let mut sparse = Workbook::default(); + sparse.set_number(cell(0, 0), 2.0); + sparse.set_number(cell(499_999, 0), 3.0); + sparse + .set_formula(cell(0, 5), "=SUM(A1:INDEX(A1:A500000,500000))") + .unwrap(); + assert_eq!(sparse.value(cell(0, 5)), Value::Number(5.0)); + assert!(sparse.statistics().dependency_edges < 20); + sparse.set_number(cell(123_456, 0), 7.0); + assert_eq!(sparse.value(cell(0, 5)), Value::Number(12.0)); + assert!(matches!( + sparse.set_formula(cell(0, 0), "=SUM(A1:INDEX(A1:A3,2))"), + Err(FormulaError::Cycle(_)) + )); + assert_eq!(sparse.value(cell(0, 0)), Value::Number(2.0)); + } +} diff --git a/crates/omasheets-corpus/src/main.rs b/crates/omasheets-corpus/src/main.rs index bb19c24..1517a50 100644 --- a/crates/omasheets-corpus/src/main.rs +++ b/crates/omasheets-corpus/src/main.rs @@ -128,6 +128,9 @@ struct OwnedSummary { /// and the number of workbooks naming each, bounded like the per-file map. unsupported_functions: BTreeMap, unsupported_reasons: BTreeMap, + syntax_failure_tokens: BTreeMap, + reference_failure_kinds: BTreeMap, + mismatch_value_kinds: BTreeMap, /// Workbooks the importer opened only after skipping sheet entries that /// have no worksheet part, and how many such entries it skipped in all. workbooks_with_skipped_sheets: u64, @@ -757,7 +760,11 @@ fn score_command( let owned_failed = owned_summary.failed; let report = ScoreReport { schema: 2, - engine: "formualizer-calamine-0.8.4", + engine: if cfg!(feature = "formualizer") { + "formualizer-calamine-0.8.4" + } else { + "disabled" + }, owned_engine: omasheets_xlsx::ENGINE_NAME, stored_value_comparison: "owned-m0", summary: ScoreSummary { @@ -815,6 +822,24 @@ fn accumulate_owned(summary: &mut OwnedSummary, owned: &OwnedProbe) { .entry(reason.clone()) .or_default() += *cells as u64; } + for (token, cells) in &report.syntax_failure_tokens { + *summary + .syntax_failure_tokens + .entry(token.clone()) + .or_default() += *cells as u64; + } + for (kind, cells) in &report.reference_failure_kinds { + *summary + .reference_failure_kinds + .entry(kind.clone()) + .or_insert(0) += *cells as u64; + } + for (kind, cells) in &report.mismatch_value_kinds { + *summary + .mismatch_value_kinds + .entry(kind.clone()) + .or_insert(0) += *cells as u64; + } if !report.skipped_sheets.is_empty() { summary.workbooks_with_skipped_sheets += 1; summary.skipped_sheets += report.skipped_sheets.len() as u64; @@ -1071,6 +1096,9 @@ mod tests { "unsupported_function".to_string(), 10 - loaded, )]), + syntax_failure_tokens: BTreeMap::new(), + reference_failure_kinds: BTreeMap::new(), + mismatch_value_kinds: BTreeMap::new(), skipped_sheets: if loaded == 8 { vec!["Module1".to_string(), "Module2".to_string()] } else { diff --git a/crates/omasheets-service/tests/native_controls.rs b/crates/omasheets-service/tests/native_controls.rs index f903033..0a3372a 100644 --- a/crates/omasheets-service/tests/native_controls.rs +++ b/crates/omasheets-service/tests/native_controls.rs @@ -9,13 +9,15 @@ struct Fixture { } impl Fixture { fn new() -> Self { + static NEXT_FIXTURE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let path = std::env::temp_dir().join(format!( - "omasheets-controls-{}-{}.omasheets", + "omasheets-controls-{}-{}-{}.omasheets", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + NEXT_FIXTURE.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); let mut f = Self { service: Service::new(|| 456), @@ -89,6 +91,54 @@ fn range(row: usize, column: usize, rows: usize, columns: usize) -> Value { json!({"row":row,"column":column,"rows":rows,"columns":columns}) } +#[test] +fn index_reference_ranges_keep_their_bound_rows_after_sort_and_reopen() { + let mut f = Fixture::new(); + f.number("A1", 30.0); + f.number("A2", 10.0); + f.number("A3", 20.0); + f.formula("B6", "=SUM(A1:INDEX(A1:A3,2))"); + assert_eq!(f.cell("B6")["value"]["value"], 40.0); + f.edit(json!({"action":"sort","range":range(0,0,3,1),"column":0,"header":false,"descending":false})); + assert_eq!(f.cell("B6")["value"]["value"], 40.0); + f.number("A3", 50.0); + assert_eq!(f.cell("B6")["value"]["value"], 60.0); + let revision = f.call(json!({"kind":"revision"})); + f.reopen(); + assert_eq!(f.call(json!({"kind":"revision"})), revision); + assert_eq!(f.cell("B6")["value"]["value"], 60.0); +} + +#[test] +fn array_and_financial_formulas_keep_stable_bindings_through_sort_and_reopen() { + let mut f = Fixture::new(); + f.number("A1", 10.0); + f.number("A2", 20.0); + f.formula("B2", "=SUM(A2*{2,3})+PV(0,1,-5)"); + f.formula("E5", "=IFERROR(SUM(#REF!:#REF!),7)"); + assert_eq!(f.cell("B2")["value"]["value"], 105.0); + f.edit( + json!({"action":"sort","range":range(0,0,2,1),"column":0,"header":false,"descending":true}), + ); + let page = f.page(); + let moved = page["cells"] + .as_array() + .unwrap() + .iter() + .find(|cell| cell["a1"] == "B1") + .unwrap(); + assert_eq!(moved["formula"], "=SUM(A1*{2,3})+PV(0,1,-5)"); + assert_eq!(moved["value"]["value"], 105.0); + assert!(moved.get("formula_projection_error").is_none()); + let revision = f.call(json!({"kind":"revision"})); + f.reopen(); + assert_eq!(f.call(json!({"kind":"revision"})), revision); + assert_eq!(f.cell("B1")["value"]["value"], 105.0); + assert_eq!(f.cell("E5")["value"]["value"], 7.0); + f.number("A1", 30.0); + assert_eq!(f.cell("B1")["value"]["value"], 155.0); +} + #[test] fn moved_formula_text_preserves_absolute_axes_and_refuses_nonrectangular_ranges() { let mut f = Fixture::new(); diff --git a/crates/omasheets-xlsx/src/lib.rs b/crates/omasheets-xlsx/src/lib.rs index 5db96c2..629550c 100644 --- a/crates/omasheets-xlsx/src/lib.rs +++ b/crates/omasheets-xlsx/src/lib.rs @@ -111,6 +111,14 @@ pub struct ScoreReport { pub unsupported_functions: BTreeMap, /// Compile-failure kinds and how many formula cells hit each. pub unsupported_reasons: BTreeMap, + /// Syntax failures grouped by a fixed token class, never formula text. + #[serde(default)] + pub syntax_failure_tokens: BTreeMap, + /// Fixed failure classes, with no source references or cell values. + #[serde(default)] + pub reference_failure_kinds: BTreeMap, + #[serde(default)] + pub mismatch_value_kinds: BTreeMap, /// Sheet entries without a worksheet part that the importer skipped. #[serde(default)] pub skipped_sheets: Vec, @@ -214,6 +222,63 @@ impl ImportedWorkbook { pub fn report(&self) -> ScoreReport { let parity = self.parity(); + let mut reference_failure_kinds = BTreeMap::new(); + for failure in &self.unsupported { + if let FormulaError::InvalidReference(reference) = &failure.error { + let kind = match reference.as_str() { + "range endpoint is number" => "numeric_range_endpoint", + "range endpoint is function" => "dynamic_range_endpoint", + "range endpoints must be references" => "non_reference_endpoint", + "range endpoints cross sheets" => "cross_sheet_range", + "" => "empty_reference", + token if token.bytes().all(|b| b.is_ascii_alphabetic() || b == b'$') => { + "column_without_row" + } + token if token.bytes().all(|b| b.is_ascii_digit() || b == b'$') => { + "row_without_column" + } + _ => "invalid_a1", + }; + *reference_failure_kinds.entry(kind.to_string()).or_insert(0) += 1; + } + } + let mut mismatch_value_kinds = BTreeMap::new(); + for (_, stored, calculated) in self.mismatched_cells() { + let kind = format!("{} -> {}", value_kind(stored), value_kind(&calculated)); + *mismatch_value_kinds.entry(kind).or_insert(0) += 1; + } + let mut syntax_failure_tokens = BTreeMap::new(); + for failure in &self.unsupported { + let FormulaError::UnexpectedToken(offset) = failure.error else { + continue; + }; + let Ok(index) = self + .source_cells + .binary_search_by_key(&failure.cell, |cell| cell.cell) + else { + continue; + }; + let Some(source) = self.source_cells[index].formula.as_deref() else { + continue; + }; + let source = source.strip_prefix('=').unwrap_or(source); + let token = match source.as_bytes().get(offset) { + Some(b'{') | Some(b'}') => "array_brace", + Some(b':') => "range_colon", + Some(b',') => "comma", + Some(b';') => "semicolon", + Some(b'!') => "sheet_separator", + Some(b'[') | Some(b']') => "square_bracket", + Some(b'@') => "implicit_intersection", + Some(b'\\') => "backslash_identifier", + Some(b'(') | Some(b')') => "parenthesis", + Some(byte) if !byte.is_ascii() => "non_ascii_identifier", + Some(byte) if byte.is_ascii_alphabetic() => "identifier", + None => "end_of_formula", + _ => "other", + }; + *syntax_failure_tokens.entry(token.to_string()).or_insert(0) += 1; + } ScoreReport { schema: 2, engine: ENGINE_NAME.into(), @@ -228,6 +293,9 @@ impl ImportedWorkbook { unsupported_formulas: parity.unsupported_formulas, unsupported_functions: self.unsupported_functions(), unsupported_reasons: self.unsupported_reasons(), + syntax_failure_tokens, + reference_failure_kinds, + mismatch_value_kinds, skipped_sheets: self.skipped_sheets.clone(), } } @@ -895,6 +963,16 @@ fn bounded_formula_error(error: &FormulaError) -> String { error.to_string().chars().take(256).collect() } +fn value_kind(value: &Value) -> &str { + match value { + Value::Blank => "blank", + Value::Number(_) => "number", + Value::Boolean(_) => "boolean", + Value::Text(_) => "text", + Value::Error(error) => error.label(), + } +} + fn values_match(stored: &Value, calculated: &Value) -> bool { match (stored, calculated) { (Value::Number(stored), Value::Number(calculated)) => { @@ -1083,6 +1161,105 @@ mod tests { assert_eq!(formula.formula.as_deref(), Some("A1+A2")); } + #[test] + fn reference_diagnostics_expose_only_fixed_failure_classes() { + let bytes = package( + &[], + "", + r#"SUM(1:2)0SUM(Data!A:A)0SUM(A1:SUM(A1:A2))01+19"#, + "", + ); + let path = temporary_xlsx(&bytes); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); + std::fs::remove_file(path).unwrap(); + assert_eq!( + report.reference_failure_kinds, + BTreeMap::from([ + ("column_without_row".into(), 1), + ("dynamic_range_endpoint".into(), 1), + ("numeric_range_endpoint".into(), 1), + ]) + ); + assert_eq!( + report.mismatch_value_kinds, + BTreeMap::from([("number -> number".into(), 1)]) + ); + } + + #[test] + fn syntax_diagnostics_expose_only_fixed_token_classes() { + let path = temporary_xlsx(&package( + &[], + "", + r#"SUM(1;2)3SUM({1,})1"#, + "", + )); + let imported = import_xlsx(&path, ImportLimits::default()).unwrap(); + std::fs::remove_file(path).unwrap(); + let report = imported.report(); + assert_eq!( + report.syntax_failure_tokens, + BTreeMap::from([("semicolon".into(), 1), ("array_brace".into(), 1),]) + ); + assert_eq!(report.unsupported_reasons["syntax"], 2); + assert_eq!(report.formula_cells_compared, 1); + } + + #[test] + fn matrix_and_database_formulas_match_independent_caches() { + let path = temporary_xlsx(&package( + &[], + "", + r#"MMULT(TRANSPOSE(A1:A2),A1:A2)13INDEX(TRANSPOSE({1,2,3;4,5,6}),3,2)6DAVERAGE({"Kind","Value";"A",10;"B",50;"A",20},"Value",{"Kind";"=A"})15DMAX({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})20DMIN({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})10DSTDEV({"Kind","Value";"A",10;"B",50;"A",20},2,{"Kind";"=A"})7.0710678118654755"#, + "", + )); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); + std::fs::remove_file(path).unwrap(); + assert_eq!(report.formula_cells_loaded, 7); + assert_eq!(report.stored_values_matched, 7); + assert_eq!(report.stored_values_mismatched, 0); + } + + #[test] + fn reference_valued_index_matches_xlsx_caches() { + let bytes = package( + &[], + "", + r#"SUM(A1:INDEX(A1:A2,2))5SUM(INDEX(A1:A2,0))5MATCH(3,INDEX(A1:A2,0),0)2"#, + "", + ); + let path = temporary_xlsx(&bytes); + let report = import_xlsx(&path, ImportLimits::default()) + .unwrap() + .report(); + std::fs::remove_file(path).unwrap(); + assert_eq!(report.formula_cells_loaded, 4); + assert_eq!(report.stored_values_matched, 4); + assert_eq!(report.stored_values_mismatched, 0); + } + + #[test] + fn array_financial_and_deleted_reference_formulas_match_xlsx_caches() { + let path = temporary_xlsx(&package( + &[], + r#"{2;3}0.1"#, + r#"SUMPRODUCT({10;20},weights)80PV(Data!rate,1,-110)100_xlfn.XLOOKUP(2,{1,2},{10,20})20SUM(#REF!:#REF!)#REF!IFERROR(SUM(A1:#REF!),17)17"#, + "", + )); + let imported = import_xlsx(&path, ImportLimits::default()).unwrap(); + std::fs::remove_file(path).unwrap(); + let report = imported.report(); + assert_eq!(report.formula_cells_observed, 6); + assert_eq!(report.formula_cells_loaded, 6); + assert_eq!(report.stored_values_matched, 6); + assert_eq!(report.stored_values_mismatched, 0); + assert!(report.unsupported_reasons.is_empty()); + } + #[test] fn shared_formulas_expand_from_their_anchor_cell() { // The shared group is anchored at B5 with ref A5:B6; A5 carries its diff --git a/docs/FUNCTIONS.md b/docs/FUNCTIONS.md index 109e3df..01a79da 100644 --- a/docs/FUNCTIONS.md +++ b/docs/FUNCTIONS.md @@ -1,22 +1,41 @@ # Supported formula functions The owned M0 engine (`crates/omasheets-calc`) accepts exactly the -96 function names listed below, grouped for reading. +110 function names listed below, grouped for reading. A test in the calc crate fails when this file and the registry disagree, so the count here is never edited by hand: add the function to the registry and regenerate this list. Operators: `+ - * / ^ & %`, unary `+`/`-`, comparisons `= <> < <= > >=`, error literals (`#REF!`, `#N/A`, `#DIV/0!`, `#VALUE!`, `#NUM!`, `#NAME?`, -`#NULL!`, and `Sheet!#REF!` for a deleted cell on another sheet), omitted arguments, bounded rectangular ranges, absolute markers, -cross-sheet references, workbook and sheet-scoped defined names (tokens past +`#NULL!`, and `Sheet!#REF!` for a deleted cell on another sheet), omitted arguments, +bounded rectangular ranges (including qualified endpoints and deleted endpoints +such as `A1:#REF!`, which evaluate to `#REF!`), absolute markers, +cross-sheet references, workbook and sheet-scoped defined names (including +`Sheet!LocalName`; tokens past the grid such as `Table1` are names), implicit intersection of a range in scalar position, and elementwise evaluation of range expressions inside aggregate arguments (`SUM(IF(A1:A5=0,0,B1:B5))`, `SUMPRODUCT((A1:A5>2)*B1:B5)`). +Rectangular array constants support numbers, text, booleans and error literals, +comma-separated columns and semicolon-separated rows, up to 1,000,000 values. +They work in aggregates, elementwise expressions and INDEX/MATCH/LOOKUP, +VLOOKUP/HLOOKUP/XLOOKUP. A scalar use takes the first value; spilling into +neighbouring cells is not implemented. `_xlfn.` and `_xlfn._xlws.` prefixes +resolve only to functions already in the registry. + +`INDEX` also returns references: `SUM(A1:INDEX(A1:A100,D1))` follows the +selector in D1, and a zero row or column selects that entire axis. The range +operator binds a bounded envelope of all possible endpoint selections. Native +replay preserves that envelope's row and column identities, including after +sorting. Constant row/column selections narrow the calculation dependencies after +stable binding, so unused source columns do not create false cycles. Dynamic axes +retain their bounded envelope; potential cycles within it are still refused. A moved formula +whose current A1 spelling cannot preserve those identities reports a projection +refusal instead of exporting different references. Deliberately unsupported: `TODAY`, `NOW`, `RAND` and every other volatile function (until the calculation context consumes stored tick events), external workbook references, -3D references, array constants and array formulas, `INDIRECT`, `OFFSET`, +3D references, spilling array formulas, `INDIRECT`, `OFFSET`, `CELL`, add-in (`_xll.`) calls, locale-sensitive parsing such as `DATEVALUE` and `TEXT`, and the 1904 date system. @@ -26,6 +45,27 @@ unsorted keys are undefined in Excel and are not promised here. ## Registry +### Matrices and databases + +- `TRANSPOSE` +- `MMULT` +- `DAVERAGE` +- `DMAX` +- `DMIN` +- `DSTDEV` + +`TRANSPOSE` and `MMULT` produce bounded arrays consumed by aggregates and +lookups; scalar uses take the first element. MMULT requires numeric, nonblank +inputs with matching inner dimensions, at most 1,000,000 output values and +50,000,000 multiply-add terms. Larger products return `#NUM!`. + +Database aggregates resolve fields by heading or one-based column number. +Criteria columns on one row are ANDed; rows are ORed. Duplicate headings, +blank criteria, text prefixes, wildcards and comparison operators are supported. +At most 50,000,000 candidate/criterion comparisons are allowed per call. +Formula criteria with nonmatching/blank headings are not implemented and return +`#VALUE!`; they require relative formula evaluation for each database record. + ### Aggregates and statistics - `SUM` @@ -49,6 +89,12 @@ unsorted keys are undefined in Excel and are not promised here. - `AVERAGEA` - `CORREL` - `NORMDIST` +- `NORM.DIST` +- `NORMSDIST` +- `NORM.S.DIST` +- `COVAR` +- `COVARIANCE.P` +- `COVARIANCE.S` ### Conditional aggregates @@ -142,6 +188,8 @@ unsorted keys are undefined in Excel and are not promised here. ### Financial - `PMT` +- `PV` +- `IRR` - `NPV` - `XNPV` - `XIRR` diff --git a/docs/NATIVE-SPREADSHEET-SUPPORT.md b/docs/NATIVE-SPREADSHEET-SUPPORT.md index 48802f0..0f69a87 100644 --- a/docs/NATIVE-SPREADSHEET-SUPPORT.md +++ b/docs/NATIVE-SPREADSHEET-SUPPORT.md @@ -56,7 +56,9 @@ The width calculation follows the [SpreadsheetML column specification](https://l ## Formula and editing boundaries -The parser registry contains 96 function names, including TEXTJOIN. The +The parser registry contains 104 function names, including TEXTJOIN, PV, IRR, +covariance and standard-normal distribution functions. Bounded array constants +work in aggregates and lookups without spilling into neighbouring cells. The registry and [function list](FUNCTIONS.md) are checked together. Clock and random functions remain explicitly refused: calculation does not read hidden time or randomness. The core's stored tick events have not been connected to volatile diff --git a/docs/V0.1-RELEASE.md b/docs/V0.1-RELEASE.md index 2b703ba..5501216 100644 --- a/docs/V0.1-RELEASE.md +++ b/docs/V0.1-RELEASE.md @@ -29,16 +29,27 @@ projections. Agents cannot append to `main`, approve a merge or publish output. | Native import | Bounded XLSX conversion stages one event transaction, publishes without replacement and verifies replay digest | Installed API proof plus dogfood workbook review confirms the manifest is sufficient | | Omarchy integration | Reproducible bundle includes the native grid, compatibility window, user-facing commands, `.omasheets` MIME routing and an explicit, reversible `setup --omarchy --enable-service` path; compiler-free acceptance exercises the grid, import, edit, branch, check, diff, merge, CSV, XLSX, Parquet, close and reopen | Installed workflow and rollback pass owner review on Omarchy | | Resource targets | Headless wiring evidence only | Declared baseline meets the M1 open, paint, scroll and idle-RSS targets | -| Corpus targets | 99.6% open, 89.26% raw parse, 99.87% stored-value match | Parse-coverage denominator is recorded and the M2 gates pass without changing the frozen sample | +| Corpus targets | 99.6% open, 89.75% raw parse, 99.87% stored-value match | Parse-coverage denominator is recorded and the M2 gates pass without changing the frozen sample | ## Required release evidence -The current frozen-sample raw parse rate is 825,016 / 924,235 (89.26%). +The frozen-sample raw parse rate is now 829,529 / 924,235 (89.75%), up +from 825,016 (89.26%) on the same sample. The 2026-09-08 sequential comparison +of `ecf0036` and `12d1c8c` compiled 4,513 additional formulas and matched +4,505 additional stored values. Total mismatches changed from 1,084 to 1,092; +stored-value agreement remains 99.87% of compared formulas. See the +[aggregate evidence](../corpus/sources/enron-figshare.score-summary.json) and +[before/after delta](../corpus/sources/enron-figshare.score-delta.json). + The 47,007 formulas classified as external references alone cap raw coverage at 94.91% while the external-workbook refusal policy remains in force, even if every other formula compiles. This is an arithmetic upper bound from the -existing aggregate summary, not a new scorer run. The roadmap's 97% target -therefore needs an explicit maintainer decision about its denominator or +aggregate summary. A further 21,737 first-failure classifications are +proprietary add-in functions, including `_XLL.HPVAL`, `INDGENCOST` and +`CALCSKEW`; implementing ordinary Excel functions cannot supply their missing +implementations. Together these classes cap raw coverage at 92.56% for this +classification of the frozen sample. This is a bound, not an adjusted score. +The roadmap's 97% target therefore needs an explicit maintainer decision about its denominator or release scope. Do not silently exclude unsupported formulas, lower the target, or describe the current corpus gate as passed. Engineering improvements to supported formulas remain useful independently of that decision.