From 7af01b73b2bf7afa27cfba9443f538696215f175 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 17:32:30 +0100 Subject: [PATCH 01/14] design(eval): perplexity harness contract + design doc (#231) Adds the DotLLM.Core.Evaluation abstractions and the design document for a shared perplexity harness, addressing the blocker recorded in upstream #416: every remaining numerics-changing CPU lever is ungated because no harness exists. The contract models the one axis the existing per-test helpers actually diverged on -- IPerplexityModel.ReturnsAllRows -- so a single evaluator can replace both the O(n) all-rows CPU shape and the O(n^2) growing-prefix last-row-only shape without changing any number either currently produces. The evaluator deliberately never loads weights and takes an already-constructed model, and the corpus is streamed rather than materialized. Both are constraints from the planned iGPU follow-on: on unified-memory parts a large VRAM carve-out leaves host RAM scarce, and paying for the model twice is the failure mode. Co-Authored-By: Claude Opus 5 (1M context) --- .../2026-07-30-perplexity-harness-design.md | 110 ++++++++++++++++++ .../Evaluation/IPerplexityModel.cs | 56 +++++++++ .../Evaluation/PerplexityResult.cs | 71 +++++++++++ 3 files changed, 237 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-30-perplexity-harness-design.md create mode 100644 src/DotLLM.Core/Evaluation/IPerplexityModel.cs create mode 100644 src/DotLLM.Core/Evaluation/PerplexityResult.cs diff --git a/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md b/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md new file mode 100644 index 00000000..9b9e9804 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md @@ -0,0 +1,110 @@ +# Perplexity harness — design + +**Issue:** jamesburton/dotLLM#231 · **Upstream driver:** kkokosa/dotLLM#416 · **Date:** 2026-07-30 + +## Problem + +Upstream #416's investigation log ends on a blocker: *"Every remaining lever changes numerics and +none of them can currently be evaluated."* Scale-granularity repacking (G=64/128, estimated +1.31–1.56×) and every other numerics-changing CPU lever are gated on an evaluation harness that +does not exist upstream. + +`upstream/main` has only `samples/DotLLM.Sample.Logprobs/Program.cs`. Our `dev` has perplexity +scoring in ten files — but entirely as **duplicated private per-test helpers**. `StableLogProb` is +copy-pasted verbatim; the same English corpus string appears in multiple files; two different +scoring strategies have diverged without a shared definition. None of it is reachable as a gate for +kernel work. + +So this is consolidation plus extension, not greenfield — but the consolidation half must be done +from `dev`, because that is where the helpers live. See Sequencing. + +## The one real axis + +The existing helpers diverged into two shapes for a single reason: **whether the backend's +`Forward` returns logits for every position or only the last one.** + +- CPU returns `[seqLen, vocab]`, so one teacher-forced prefill scores every next-token NLL — O(n). + (`BitNetAccuracyTests.Cpu_Perplexity_OnFixedPassage_IsSane`) +- CUDA returns only the final row, so each target needs its own growing-prefix re-prefill — O(n²), + which is why those helpers carry a stride to keep the sweep brisk. + (`CudaFlashPrefillForwardHarness.PrefillGrowingPrefixPerplexity`, and its twin in + `CudaG3PrefillForwardHarness`) + +Everything else about them is identical. Modelling that axis explicitly — `ReturnsAllRows` on +`IPerplexityModel` — is what lets one evaluator replace both without changing any number either +currently produces. + +## Components + +**`DotLLM.Core.Evaluation`** (abstractions only, so `main` and `dev` can both reference the +contract without dragging in the implementation): + +- `IPerplexityModel` — `VocabSize`, `MaxContextLength`, `ReturnsAllRows`, `Forward(tokens, positions)`. + Deliberately narrower than `IModel`: perplexity needs no sampling, no KV-cache lifetime + management, no streaming, and binding to the full interface would prevent scoring a bare backend + or a test double. +- `PerplexityMode` — `TeacherForced` | `SlidingWindow`. +- `PerplexityOptions(Mode, ContextLength, Stride, MaxTokens)`. +- `PerplexityResult(Perplexity, MeanNegativeLogLikelihood, ScoredTokens, WindowCount)`. + +**`PerplexityEvaluator`** — strategy selected from `ReturnsAllRows`, not from the caller. + +**Corpus handling** — streamed and tokenized in chunks. + +**CLI verb** — context length, stride, corpus path, token cap. + +## Two modes, two purposes + +`TeacherForced` preserves the in-tree "G1 precedent" methodology exactly. It is **ratio-oriented**: +the load-bearing signal is the OFF/ON perplexity ratio on identical tokens under a <1% gate, not +the absolute value. Not comparable to published figures, and must not be presented as if it were. + +`SlidingWindow` is new and **absolute-value oriented**: windows of `ContextLength` advanced by +`Stride`, scoring only tokens beyond the carried-over prefix so every scored token has full-length +context. Matches llama.cpp's `--perplexity` methodology so figures compare directly to published +numbers. + +## Memory constraint (from planned Track D) + +On Strix Halo's UMA, a large VRAM carve-out leaves host RAM scarce, and perplexity is the workload +most punished by it — a long sequence of full-context prefills rather than a single load. A harness +that mmaps weights host-side while the backend separately uploads them pays for the model twice +against an already-halved budget. + +Therefore, as a design constraint rather than a later optimisation: + +- **The evaluator never loads weights.** It takes an already-constructed `IPerplexityModel`. +- **The corpus is streamed and tokenized in chunks**, never materialized whole. + +Both cost nothing now and keep Track D a pure optimisation rather than a rewrite. + +## Verification + +1. **Consolidation is behaviour-preserving.** Migrated tests must produce *numerically identical* + results to their previous private helpers. A changed number means the consolidation altered + semantics — this is the primary regression gate, and it is the reason `TeacherForced` is + preserved verbatim rather than "improved" in passing. +2. **`SlidingWindow` is genuinely comparable.** Validated against a published llama.cpp perplexity + figure on matching model, corpus, context length and stride, within a stated tolerance. Without + this the word "comparable" is unearned, and the harness would give upstream false confidence on + exactly the numerics-changing decisions it is meant to gate. +3. **`ScoredTokens` is reported and checked.** A perplexity over a different token count is a + different measurement; cross-run comparison requires it to match. + +## Sequencing + +Built from `main` in a worktree (`issue/231-perplexity-harness`), PR targets upstream `main`, then +merges to `dev`. + +Because the ten duplicated helpers exist only on `dev`, the split is: + +- **On `main` / this PR:** the contract, the evaluator, corpus handling, CLI verb, and validation + of `SlidingWindow` against llama.cpp. Self-contained and upstream-contributable. +- **On `dev`, follow-up:** migrate the ten existing helpers onto the harness, gated on producing + identical numbers. + +## Out of scope + +Track D (iGPU optimisation of the harness under a large VRAM carve-out, including eliminating +host+device double-loading) is deliberately deferred until this and #233 land. The constraints +above exist so D is optimisation, not rework. diff --git a/src/DotLLM.Core/Evaluation/IPerplexityModel.cs b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs new file mode 100644 index 00000000..fd7f0a00 --- /dev/null +++ b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs @@ -0,0 +1,56 @@ +using DotLLM.Core.Tensors; + +namespace DotLLM.Core.Evaluation; + +/// +/// The minimal model surface perplexity scoring needs: a teacher-forced forward pass over a +/// token window, plus enough metadata to interpret the logits it returns. +/// +/// +/// Deliberately narrower than IModel. Perplexity needs neither sampling, KV-cache +/// lifetime management, nor streaming, and binding the evaluator to the full model interface +/// would prevent scoring a bare backend or a test double. +/// The evaluator never loads weights. Implementations are handed an +/// already-constructed model, so a caller that has the weights resident on a device is never +/// forced into a second host-side copy. This matters on unified-memory parts (e.g. Strix Halo) +/// where a large VRAM carve-out leaves host RAM scarce, and perplexity — a long sequence of +/// full-context prefills rather than a single load — is the workload most punished by paying +/// for the model twice. +/// +public interface IPerplexityModel +{ + /// Vocabulary size; the row length of the returned logits. + int VocabSize { get; } + + /// + /// Maximum token window a single call accepts. Sliding-window scoring + /// never requests a window larger than this. + /// + int MaxContextLength { get; } + + /// + /// when returns logits for every position + /// (shape [seqLen, VocabSize]); when only the final row is + /// returned (shape [1, VocabSize] or [VocabSize]). + /// + /// + /// This is the single axis that decides scoring cost, and the reason the existing per-test + /// helpers diverged into two shapes. All-rows backends score a window in one forward pass + /// (O(n)); last-row-only backends must re-prefill each growing prefix (O(n^2)). The evaluator + /// selects the strategy from this flag rather than the caller hard-coding one. + /// + bool ReturnsAllRows { get; } + + /// + /// Runs a teacher-forced forward pass over . + /// + /// Input token ids for this window. + /// + /// Absolute position ids, one per token. Passed explicitly rather than derived, so a sliding + /// window can score at its true absolute positions instead of restarting at zero. + /// + /// + /// Logits, owned by the caller. Row layout is governed by . + /// + ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions); +} diff --git a/src/DotLLM.Core/Evaluation/PerplexityResult.cs b/src/DotLLM.Core/Evaluation/PerplexityResult.cs new file mode 100644 index 00000000..c6a3661b --- /dev/null +++ b/src/DotLLM.Core/Evaluation/PerplexityResult.cs @@ -0,0 +1,71 @@ +namespace DotLLM.Core.Evaluation; + +/// +/// Scoring strategy. Chosen by the caller; the evaluator picks the execution path from +/// . +/// +public enum PerplexityMode +{ + /// + /// Teacher-forced scoring over one window per stride step, each scored from a single forward + /// pass where the backend permits it. The established in-tree methodology (the "G1 precedent" + /// referenced by the CUDA prefill harnesses); preserved so existing quality gates keep their + /// meaning after consolidation. + /// + /// + /// Ratio-oriented: the load-bearing signal is the OFF/ON perplexity ratio on identical tokens, + /// not the absolute value. Not comparable to published figures. + /// + TeacherForced, + + /// + /// Sliding-window scoring matching llama.cpp's --perplexity methodology: the corpus is + /// walked in windows of advanced by + /// , and only tokens beyond the carried-over prefix are + /// scored, so every scored token has a full-length context. + /// + /// + /// Absolute-value oriented: comparable to published llama.cpp figures when context length, + /// stride, corpus and model match. + /// + SlidingWindow, +} + +/// Configuration for a perplexity run. +/// Scoring strategy. +/// +/// Window size in tokens. Clamped to . +/// +/// +/// Tokens advanced between windows. Equal to gives +/// non-overlapping windows (llama.cpp's default); a smaller value overlaps, scoring each token +/// with more preceding context at proportionally higher cost. +/// +/// +/// Upper bound on corpus tokens consumed; 0 means unbounded. Bounds runtime on large +/// corpora without truncating the corpus file itself. +/// +public readonly record struct PerplexityOptions( + PerplexityMode Mode, + int ContextLength, + int Stride, + int MaxTokens = 0); + +/// Outcome of a perplexity run. +/// +/// exp(MeanNegativeLogLikelihood) — the headline figure. +/// +/// +/// Mean NLL in nats over all scored tokens. Reported alongside perplexity because differences +/// between near-identical runs are easier to read here than through the exponential. +/// +/// +/// Number of tokens that contributed. Comparisons across runs are meaningful only when this +/// matches: a perplexity computed over a different token count is a different measurement. +/// +/// Number of forward windows evaluated. +public readonly record struct PerplexityResult( + double Perplexity, + double MeanNegativeLogLikelihood, + int ScoredTokens, + int WindowCount); From 2cd9a62430b3f3e5560fa1cd02df53d629f64925 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:16:02 +0100 Subject: [PATCH 02/14] plan(eval): perplexity harness implementation plan (#231) Nine TDD tasks. Pins down what "llama.cpp-comparable" actually means before any code: window w covers [w*S, w*S+L) and scores absolute targets in [w*S+L-S, w*S+L), so scored ranges tile the corpus exactly and every scored token carries L-S tokens of context. S = L/2 reproduces llama.cpp's default. Co-Authored-By: Claude Opus 5 (1M context) --- .../plans/2026-07-30-perplexity-harness.md | 956 ++++++++++++++++++ 1 file changed, 956 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-30-perplexity-harness.md diff --git a/docs/superpowers/plans/2026-07-30-perplexity-harness.md b/docs/superpowers/plans/2026-07-30-perplexity-harness.md new file mode 100644 index 00000000..d53ed34d --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-perplexity-harness.md @@ -0,0 +1,956 @@ +# Perplexity Harness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a shared perplexity harness with a llama.cpp-comparable sliding-window mode, so numerics-changing kernel work (ours and upstream's) has an evaluation gate. + +**Architecture:** Abstractions in `DotLLM.Core.Evaluation` (already landed, commit `7af01b73`); a single `PerplexityEvaluator` that selects its execution strategy from `IPerplexityModel.ReturnsAllRows` rather than from the caller; a streaming corpus reader; a `TransformerModel` adapter; and a `dotllm perplexity` CLI verb. + +**Tech Stack:** .NET 10, C#, xUnit (+ `Xunit.SkippableFact`), Spectre.Console `CommandApp` for CLI. + +## Global Constraints + +- Branch `issue/231-perplexity-harness`, worktree `.claude/worktrees/ppl-harness`, based on `main`. PR targets upstream `main`. +- File-scoped namespaces. `enable`. XML doc comments on all public APIs. +- Commit messages include `(#231)` and end with `Co-Authored-By: Claude Opus 5 (1M context) `. +- **The evaluator never loads weights.** It accepts an already-constructed `IPerplexityModel`. Non-negotiable: it is the Track D constraint. +- **The corpus is streamed and tokenized in chunks**, never materialized whole. +- `TeacherForced` mode must be preserved verbatim in behaviour. It is the "G1 precedent" methodology that existing quality gates depend on; changing any number it produces defeats the migration gate. +- No `System.Linq` on scoring hot paths; no managed allocation per scored token. + +--- + +## Methodology note: what "llama.cpp-comparable" means precisely + +llama.cpp's `perplexity` walks the corpus in chunks of `n_ctx` and scores **only the second half** of each chunk, so every scored token has at least `n_ctx/2` tokens of preceding context. The first half of the very first chunk is never scored. + +Generalised to `(ContextLength L, Stride S)`: + +- Window `w` covers absolute token range `[w*S, w*S + L)`. +- Scored targets are absolute indices `t` in `[w*S + L - S, w*S + L)`. +- Each scored token therefore has `L - S` tokens of context, and consecutive windows tile the scored tokens exactly — no gaps, no double-counting. +- **`S = L/2` reproduces llama.cpp's default exactly.** + +A forward pass over window `[s, s+L)` returns rows `0..L-1`, where row `i` holds the distribution predicting the token at absolute index `s+i+1`. So scoring absolute target `t` reads row `t-s-1`. This requires `L - S >= 1`. + +Tokens before the first window's scored range are never scored, matching llama.cpp. This is a known small bias and is documented rather than silently "fixed", because fixing it would break comparability. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `src/DotLLM.Core/Evaluation/IPerplexityModel.cs` | **Landed.** Model contract. | +| `src/DotLLM.Core/Evaluation/PerplexityResult.cs` | **Landed.** `PerplexityMode`, `PerplexityOptions`, `PerplexityResult`. | +| `src/DotLLM.Engine/Evaluation/LogProb.cs` | Numerically stable log-softmax of a single logit row. | +| `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` | Both modes; strategy chosen from `ReturnsAllRows`. | +| `src/DotLLM.Engine/Evaluation/CorpusReader.cs` | Streaming corpus → token chunks. | +| `src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs` | `IPerplexityModel` over `TransformerModel`. | +| `src/DotLLM.Cli/Commands/PerplexityCommand.cs` | `dotllm perplexity` verb. | +| `tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs` | Deterministic test double. | +| `tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs` | Log-softmax correctness. | +| `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` | Both modes, both backend shapes. | +| `tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs` | Streaming/chunking. | +| `tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs` | llama.cpp figure validation. | + +--- + +### Task 1: Stable log-probability + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/LogProb.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs` + +**Interfaces:** +- Consumes: nothing. +- Produces: `static double LogProb.OfTarget(ReadOnlySpan logits, int target)` — log-softmax value at `target`, in nats. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class LogProbTests +{ + [Fact] + public void OfTarget_UniformLogits_IsNegativeLogVocab() + { + // Four equal logits => p = 1/4 for each => log p = -log 4. + var logits = new float[] { 2.5f, 2.5f, 2.5f, 2.5f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.Equal(-Math.Log(4.0), actual, 12); + } + + [Fact] + public void OfTarget_IsShiftInvariant() + { + var a = new float[] { 1f, 2f, 3f }; + var b = new float[] { 1001f, 1002f, 1003f }; + Assert.Equal(LogProb.OfTarget(a, 1), LogProb.OfTarget(b, 1), 12); + } + + [Fact] + public void OfTarget_LargeLogits_DoesNotOverflow() + { + // Naive exp() would overflow to infinity here; the max-shift must prevent it. + var logits = new float[] { 800f, 900f, 1000f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.True(double.IsFinite(actual)); + Assert.Equal(0.0, actual, 6); // target dominates => p ~ 1 => log p ~ 0 + } + + [Fact] + public void OfTarget_SumOfProbabilitiesIsOne() + { + var logits = new float[] { -1.5f, 0.25f, 3f, 0.5f }; + double sum = 0; + for (int i = 0; i < logits.Length; i++) sum += Math.Exp(LogProb.OfTarget(logits, i)); + Assert.Equal(1.0, sum, 10); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~LogProbTests"` +Expected: FAIL — `LogProb` does not exist (compile error). + +- [ ] **Step 3: Write minimal implementation** + +```csharp +namespace DotLLM.Engine.Evaluation; + +/// Numerically stable log-softmax over a single row of logits. +public static class LogProb +{ + /// + /// Returns log P(target) in nats under a softmax over . + /// + /// + /// Uses the max-shift identity log softmax(x)_t = (x_t - m) - log sum_j exp(x_j - m) + /// with m = max(x), so no exp argument is ever positive and overflow is + /// impossible. Accumulates in : a vocab of 128k float32 terms loses + /// meaningful precision in float32, and perplexity differences between near-identical runs + /// are exactly what this harness exists to resolve. + /// + public static double OfTarget(ReadOnlySpan logits, int target) + { + if ((uint)target >= (uint)logits.Length) + throw new ArgumentOutOfRangeException(nameof(target)); + + float max = logits[0]; + for (int j = 1; j < logits.Length; j++) + if (logits[j] > max) max = logits[j]; + + double sumExp = 0; + for (int j = 0; j < logits.Length; j++) + sumExp += Math.Exp(logits[j] - max); + + return (logits[target] - max) - Math.Log(sumExp); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~LogProbTests"` +Expected: PASS, 4 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/LogProb.cs tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs +git commit -m "feat(eval): numerically stable log-softmax for perplexity scoring (#231)" +``` + +--- + +### Task 2: Test double + +**Files:** +- Create: `tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs` + +**Interfaces:** +- Consumes: `IPerplexityModel` from `DotLLM.Core.Evaluation`. +- Produces: `FakePerplexityModel(int vocabSize, int maxContextLength, bool returnsAllRows, Func rowFactory)` with `IReadOnlyList ForwardCalls` recording every window passed to `Forward`. + +The recording matters as much as the logits: Tasks 3–5 assert on *which windows were evaluated*, which is how window tiling and the O(n) vs O(n²) strategy split get tested without a real model. + +- [ ] **Step 1: Write the implementation** (a test double, so no test-first cycle) + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Tests.Unit.Evaluation; + +/// +/// Deterministic for evaluator tests. Records every window it is +/// asked to score so tests can assert on window tiling, not just on the resulting number. +/// +internal sealed class FakePerplexityModel : IPerplexityModel, IDisposable +{ + private readonly Func _rowFactory; // (absolutePosition, vocabSize) => logits + private readonly List _forwardCalls = new(); + private readonly List _issued = new(); + + public FakePerplexityModel( + int vocabSize, int maxContextLength, bool returnsAllRows, + Func rowFactory) + { + VocabSize = vocabSize; + MaxContextLength = maxContextLength; + ReturnsAllRows = returnsAllRows; + _rowFactory = rowFactory; + } + + public int VocabSize { get; } + public int MaxContextLength { get; } + public bool ReturnsAllRows { get; } + + /// Token windows passed to , in call order. + public IReadOnlyList ForwardCalls => _forwardCalls; + + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + { + _forwardCalls.Add(tokens.ToArray()); + + int rows = ReturnsAllRows ? tokens.Length : 1; + int firstRow = ReturnsAllRows ? 0 : tokens.Length - 1; + var tensor = UnmanagedTensor.Allocate(new TensorShape(rows, VocabSize), DType.F32); + unsafe + { + var dest = new Span((void*)tensor.DataPointer, rows * VocabSize); + for (int r = 0; r < rows; r++) + _rowFactory(positions[firstRow + r], VocabSize).CopyTo(dest[(r * VocabSize)..]); + } + _issued.Add(tensor); + return tensor; + } + + /// Uniform logits: every target scores exactly -log(vocabSize). + public static Func Uniform => (_, vocab) => new float[vocab]; + + public void Dispose() + { + foreach (var t in _issued) t.Dispose(); + _issued.Clear(); + } +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `dotnet build tests/DotLLM.Tests.Unit` +Expected: build succeeds. If `UnmanagedTensor.Allocate` differs in signature, adjust to the actual factory in `src/DotLLM.Core/Tensors/UnmanagedTensor.cs` — do not change the tensor type used. + +- [ ] **Step 3: Commit** + +```bash +git add tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs +git commit -m "test(eval): deterministic IPerplexityModel double recording forward windows (#231)" +``` + +--- + +### Task 3: `TeacherForced` mode, all-rows backend + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: `LogProb.OfTarget`, `IPerplexityModel`, `PerplexityOptions`, `PerplexityResult`, `FakePerplexityModel`. +- Produces: `PerplexityResult PerplexityEvaluator.Evaluate(IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options)`. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class PerplexityEvaluatorTests +{ + private static readonly int[] Tokens = Enumerable.Range(0, 32).ToArray(); + + [Fact] + public void TeacherForced_AllRowsBackend_UniformLogitsGivesVocabSizePerplexity() + { + // Uniform logits => P(any target) = 1/vocab => perplexity == vocab, exactly. + using var model = new FakePerplexityModel( + vocabSize: 7, maxContextLength: 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, ContextLength: 32, Stride: 32)); + + Assert.Equal(7.0, result.Perplexity, 9); + Assert.Equal(31, result.ScoredTokens); // n-1 targets from one pass + } + + [Fact] + public void TeacherForced_AllRowsBackend_UsesASingleForwardPass() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // The whole point of ReturnsAllRows: one pass scores every target. + Assert.Single(model.ForwardCalls); + Assert.Equal(32, model.ForwardCalls[0].Length); + } + + [Fact] + public void MeanNll_AndPerplexity_AreConsistent() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + Assert.Equal(result.Perplexity, Math.Exp(result.MeanNegativeLogLikelihood), 9); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — `PerplexityEvaluator` does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Engine.Evaluation; + +/// +/// Computes perplexity over a token sequence using an . +/// +/// +/// The evaluator never loads weights — callers pass an already-constructed model. On unified-memory +/// parts a large VRAM carve-out leaves host RAM scarce, and perplexity (a long run of full-context +/// prefills) is the workload most punished by holding a second host-side copy of the weights. +/// +public static class PerplexityEvaluator +{ + /// Scores and returns the aggregate result. + public static PerplexityResult Evaluate( + IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options) + { + ArgumentNullException.ThrowIfNull(model); + if (tokens.Length < 2) + throw new ArgumentException("At least two tokens are required to score one target.", nameof(tokens)); + + int context = Math.Min(options.ContextLength, model.MaxContextLength); + if (context < 2) + throw new ArgumentException("Context length must be at least 2.", nameof(options)); + + return options.Mode switch + { + PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), + _ => throw new NotSupportedException($"Mode {options.Mode} is not implemented yet."), + }; + } + + private static unsafe PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + Span positions = length <= 512 ? stackalloc int[length] : new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + + using ITensor logits = model.Forward(tokens[..length], positions); + int vocab = model.VocabSize; + // Row i predicts token i+1, so the final row has no target within the window. + for (int i = 0; i < length - 1; i++) + { + var row = new ReadOnlySpan((void*)(logits.DataPointer + (nint)i * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(row, tokens[i + 1]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: 1); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): PerplexityEvaluator teacher-forced mode for all-rows backends (#231)" +``` + +--- + +### Task 4: `TeacherForced` mode, last-row-only backend + +**Files:** +- Modify: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: everything from Task 3. +- Produces: no new public surface. `Evaluate` now honours `ReturnsAllRows == false` via growing-prefix re-prefill. + +This is the O(n²) path the CUDA harnesses need. It must produce the **same number** as the all-rows path on the same tokens — that equivalence is what makes the migration gate meaningful. + +- [ ] **Step 1: Write the failing test** + +```csharp + [Fact] + public void TeacherForced_LastRowOnlyBackend_MatchesAllRowsBackendExactly() + { + // Position-dependent but deterministic logits, so a wrong row/position mapping shows up. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + for (int j = 0; j < vocab; j++) row[j] = (float)Math.Sin((position + 1) * (j + 1) * 0.37); + return row; + } + + using var allRows = new FakePerplexityModel(7, 64, returnsAllRows: true, Rows); + using var lastRow = new FakePerplexityModel(7, 64, returnsAllRows: false, Rows); + var options = new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32); + + var a = PerplexityEvaluator.Evaluate(allRows, Tokens, options); + var b = PerplexityEvaluator.Evaluate(lastRow, Tokens, options); + + Assert.Equal(a.Perplexity, b.Perplexity, 9); + Assert.Equal(a.ScoredTokens, b.ScoredTokens); + } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_ReprefixesGrowingWindows() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: false, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // One forward per scored target, each one token longer than the last. + Assert.Equal(31, model.ForwardCalls.Count); + for (int i = 0; i < model.ForwardCalls.Count; i++) + Assert.Equal(i + 1, model.ForwardCalls[i].Length); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — last-row-only path reads row 0 as if it were the full grid, so perplexity mismatches (or an index error). + +- [ ] **Step 3: Write minimal implementation** + +Replace `EvaluateTeacherForced` with a dispatcher plus the two strategies: + +```csharp + private static PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + => model.ReturnsAllRows + ? TeacherForcedSinglePass(model, tokens, context) + : TeacherForcedGrowingPrefix(model, tokens, context); + + // Backend returns only the final row, so each target needs its own prefill over the growing + // prefix. O(n^2) in forward passes — unavoidable, and the reason the CUDA harnesses that + // originated this methodology carry a stride. + private static unsafe PerplexityResult TeacherForcedGrowingPrefix( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + int vocab = model.VocabSize; + var positions = new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + for (int prefix = 1; prefix < length; prefix++) + { + using ITensor logits = model.Forward(tokens[..prefix], positions.AsSpan(0, prefix)); + var row = new ReadOnlySpan((void*)logits.DataPointer, vocab); + sumNll += -LogProb.OfTarget(row, tokens[prefix]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); + } +``` + +Rename the Task 3 body to `TeacherForcedSinglePass` (signature unchanged). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 5 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): growing-prefix teacher-forced path for last-row-only backends (#231)" +``` + +--- + +### Task 5: `SlidingWindow` mode + +**Files:** +- Modify: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: everything above. +- Produces: no new public surface. `PerplexityMode.SlidingWindow` becomes functional. + +Implement exactly the tiling defined in the methodology note at the top of this plan. Re-read it before writing code. + +- [ ] **Step 1: Write the failing test** + +```csharp + [Fact] + public void SlidingWindow_TilesScoredTokensWithoutGapsOrOverlap() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16, S=8 => windows start at 0, 8, 16, 24; each scores its last 8 targets. + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 8)); + + Assert.Equal(4, result.WindowCount); + Assert.Equal(32, result.ScoredTokens); // 4 windows x 8 targets + Assert.Equal(7.0, result.Perplexity, 9); // uniform logits + + Assert.Equal(4, model.ForwardCalls.Count); + Assert.All(model.ForwardCalls, w => Assert.Equal(16, w.Length)); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(8, model.ForwardCalls[1][0]); + Assert.Equal(16, model.ForwardCalls[2][0]); + Assert.Equal(24, model.ForwardCalls[3][0]); + } + + [Fact] + public void SlidingWindow_ScoresEachTargetAtItsTrueAbsolutePosition() + { + // Logits keyed to absolute position: a window that restarts positions at zero scores + // different values and fails this. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[position % vocab] = 10f; + return row; + } + + var tokens = new int[40]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = (i + 1) % 7; // target == predicted argmax + + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, Rows); + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, 8)); + + // Row i of window [s, s+L) sits at absolute position s+i and predicts token s+i+1, + // whose id is (s+i+1)%7 -- exactly the argmax. So NLL is near zero throughout. + Assert.True(result.MeanNegativeLogLikelihood < 0.01, + $"expected confident predictions, got mean NLL {result.MeanNegativeLogLikelihood}"); + } + + [Fact] + public void SlidingWindow_RejectsStrideNotSmallerThanContext() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + Assert.Throws(() => PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16))); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — `NotSupportedException` from the `Evaluate` switch. + +- [ ] **Step 3: Write minimal implementation** + +Add the `SlidingWindow` arm to the `Evaluate` switch, then: + +```csharp + // llama.cpp `--perplexity` tiling. Window w covers [w*S, w*S + L) and scores absolute targets + // in [w*S + L - S, w*S + L), so every scored token carries L-S tokens of context and the + // scored ranges tile the corpus exactly. S = L/2 reproduces llama.cpp's default. + // Targets before the first window's scored range are never scored -- llama.cpp skips them too, + // and "fixing" that would break comparability. + private static unsafe PerplexityResult EvaluateSlidingWindow( + IPerplexityModel model, ReadOnlySpan tokens, int context, int stride) + { + if (stride < 1 || stride >= context) + throw new ArgumentException( + $"Stride must be in [1, {context - 1}] for a context of {context}; each scored token needs at least one token of context.", + nameof(stride)); + + int vocab = model.VocabSize; + var positions = new int[context]; + double sumNll = 0; + int scored = 0, windows = 0; + + for (int start = 0; start + context <= tokens.Length; start += stride) + { + for (int i = 0; i < context; i++) positions[i] = start + i; + + using ITensor logits = model.Forward(tokens.Slice(start, context), positions); + windows++; + + // Absolute targets [start + context - stride, start + context); row for target t is t-start-1. + for (int t = start + context - stride; t < start + context; t++) + { + int row = t - start - 1; + var span = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)row * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(span, tokens[t]); + scored++; + } + } + + if (scored == 0) + throw new ArgumentException( + $"Corpus of {tokens.Length} tokens is shorter than one context window of {context}.", nameof(tokens)); + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, windows); + } +``` + +Note: the last-row-only backend is **not** supported in `SlidingWindow` — throw `NotSupportedException` with a message pointing at `TeacherForced`, since re-prefilling per target inside a sliding window is both O(n²) and redundant with the growing-prefix mode. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 8 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): llama.cpp-comparable sliding-window perplexity mode (#231)" +``` + +--- + +### Task 6: Streaming corpus reader + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/CorpusReader.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs` + +**Interfaces:** +- Consumes: `DotLLM.Tokenizers` `ITokenizer` (`int[] Encode(string)`). +- Produces: `static IEnumerable CorpusReader.StreamTokens(TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536)`. + +Streaming is a spec constraint, not an optimisation: `wiki.test.raw` is ~1.3 MB and its token array is ~340k ints, and Track D's premise is that host RAM is the scarce resource. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class CorpusReaderTests +{ + // One token per whitespace-separated word; ids are word lengths, so order is checkable. + private sealed class WordTokenizer : ITokenizer + { + public int[] Encode(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(w => w.Length).ToArray(); + public string Decode(ReadOnlySpan ids) => throw new NotSupportedException(); + } + + [Fact] + public void StreamTokens_ProducesTokensInOrder() + { + using var reader = new StringReader("a bb ccc dddd"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer()).ToArray(); + Assert.Equal(new[] { 1, 2, 3, 4 }, tokens); + } + + [Fact] + public void StreamTokens_HonoursMaxTokens() + { + using var reader = new StringReader("a bb ccc dddd eeeee"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 3).ToArray(); + Assert.Equal(new[] { 1, 2, 3 }, tokens); + } + + [Fact] + public void StreamTokens_DoesNotSplitTokensAcrossChunkBoundaries() + { + // A tiny chunk size forces the boundary case: "ccc" must not become "c" + "cc". + using var reader = new StringReader("a bb ccc dddd eeeee ffffff"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 0, charChunkSize: 4).ToArray(); + Assert.Equal(new[] { 1, 2, 3, 4, 5, 6 }, tokens); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~CorpusReaderTests"` +Expected: FAIL — `CorpusReader` does not exist. If `ITokenizer`'s real shape differs, adjust the stub to match `src/DotLLM.Tokenizers/` and keep the assertions. + +- [ ] **Step 3: Write minimal implementation** + +```csharp +using System.Text; +using DotLLM.Tokenizers; + +namespace DotLLM.Engine.Evaluation; + +/// Streams a text corpus into tokens without materializing the whole file or token array. +public static class CorpusReader +{ + /// + /// Reads in character chunks, tokenizes each chunk, and yields token + /// ids in order, stopping after (0 = unbounded). + /// + /// + /// Chunks are cut at the last whitespace so a token is never split across a boundary; the + /// remainder is carried into the next chunk. The final chunk is flushed whole. + /// + public static IEnumerable StreamTokens( + TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(tokenizer); + if (charChunkSize < 1) throw new ArgumentOutOfRangeException(nameof(charChunkSize)); + + var buffer = new char[charChunkSize]; + var carry = new StringBuilder(); + int emitted = 0; + + while (true) + { + int read = reader.Read(buffer, 0, buffer.Length); + if (read == 0) break; + + carry.Append(buffer, 0, read); + string pending = carry.ToString(); + + int cut = pending.LastIndexOf(' '); + if (cut < 0) continue; // no safe split point yet; accumulate + + string ready = pending[..cut]; + carry.Clear(); + carry.Append(pending[(cut + 1)..]); + + foreach (int id in tokenizer.Encode(ready)) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + + if (carry.Length > 0) + { + foreach (int id in tokenizer.Encode(carry.ToString())) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~CorpusReaderTests"` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/CorpusReader.cs tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs +git commit -m "feat(eval): streaming corpus tokenizer for perplexity runs (#231)" +``` + +--- + +### Task 7: `TransformerModel` adapter + +**Files:** +- Create: `src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs` + +**Interfaces:** +- Consumes: `TransformerModel` (`Config.VocabSize`, `Config.MaxSequenceLength`, `ITensor Forward(ReadOnlySpan, ReadOnlySpan, int deviceId)`), `IPerplexityModel`. +- Produces: `TransformerPerplexityModel(TransformerModel model, int deviceId = -1)` implementing `IPerplexityModel`. + +- [ ] **Step 1: Write the implementation** + +`TransformerModel.Forward` is documented as returning `[seqLen, vocab]` for all input positions, so `ReturnsAllRows` is `true`. + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; +using DotLLM.Models.Architectures; + +namespace DotLLM.Models.Evaluation; + +/// Adapts to . +/// +/// Holds a borrowed reference: the adapter does not own the model and does not dispose it, so the +/// caller keeps a single resident copy of the weights. This is the whole point of the evaluator +/// taking a constructed model rather than a path. +/// +public sealed class TransformerPerplexityModel : IPerplexityModel +{ + private readonly TransformerModel _model; + private readonly int _deviceId; + + /// An already-loaded model. Not owned; not disposed by this adapter. + /// Device for the forward pass; -1 is CPU. + public TransformerPerplexityModel(TransformerModel model, int deviceId = -1) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _deviceId = deviceId; + } + + /// + public int VocabSize => _model.Config.VocabSize; + + /// + public int MaxContextLength => _model.Config.MaxSequenceLength; + + /// + public bool ReturnsAllRows => true; + + /// + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + => _model.Forward(tokens, positions, _deviceId); +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `dotnet build src/DotLLM.Models` +Expected: build succeeds. + +- [ ] **Step 3: Commit** + +```bash +git add src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs +git commit -m "feat(eval): IPerplexityModel adapter over TransformerModel (#231)" +``` + +--- + +### Task 8: `dotllm perplexity` CLI verb + +**Files:** +- Create: `src/DotLLM.Cli/Commands/PerplexityCommand.cs` +- Modify: `src/DotLLM.Cli/Program.cs` (register alongside `run`/`chat`/`serve`) + +**Interfaces:** +- Consumes: `PerplexityEvaluator.Evaluate`, `CorpusReader.StreamTokens`, `TransformerPerplexityModel`, `GgufFileResolver` (follow `RunCommand`'s resolution pattern exactly). +- Produces: CLI verb `dotllm perplexity --corpus [--context N] [--stride N] [--max-tokens N] [--mode sliding-window|teacher-forced]`. + +- [ ] **Step 1: Write the command** + +Follow `RunCommand`'s `AsyncCommand` shape. Defaults: `--context 512`, `--stride 256` (i.e. `L/2`, llama.cpp's default), `--mode sliding-window`, `--max-tokens 0`. + +Output must report `Perplexity`, `MeanNegativeLogLikelihood`, `ScoredTokens`, `WindowCount`, plus the effective context and stride — a perplexity figure without its token count and window geometry is not comparable to anything, which is the failure this harness exists to prevent. + +- [ ] **Step 2: Register the verb in `Program.cs`** + +```csharp +config.AddCommand("perplexity") + .WithDescription("Compute perplexity over a text corpus.") + .WithExample("perplexity", "QuantFactory/SmolLM-135M-GGUF", "--corpus", "wiki.test.raw", "--context", "512", "--stride", "256"); +``` + +- [ ] **Step 3: Verify end-to-end on a small real model** + +Run: `dotnet run --project src/DotLLM.Cli -- perplexity QuantFactory/SmolLM-135M-GGUF --corpus --context 256 --stride 128 --max-tokens 2048` +Expected: a finite perplexity in a plausible range for a 135M model on English prose (roughly 15–60), with `ScoredTokens` and `WindowCount` consistent with the tiling — `WindowCount == floor((tokens - context)/stride) + 1` and `ScoredTokens == WindowCount * stride`. + +- [ ] **Step 4: Commit** + +```bash +git add src/DotLLM.Cli/Commands/PerplexityCommand.cs src/DotLLM.Cli/Program.cs +git commit -m "feat(cli): dotllm perplexity verb (#231)" +``` + +--- + +### Task 9: llama.cpp comparability validation + +**Files:** +- Create: `tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs` + +**Interfaces:** +- Consumes: all of the above. +- Produces: the acceptance evidence for the spec's second verification requirement. + +Without this the word "comparable" is unearned, and the harness would give upstream false confidence on exactly the numerics-changing decisions it is meant to gate. + +- [ ] **Step 1: Produce the reference figure** + +Run llama.cpp's own perplexity on a fixed model + corpus, recording the exact build, model file, quantization, context and stride: + +```bash +llama-perplexity -m -f wiki.test.raw -c 512 +``` + +Record the reported perplexity, chunk count, and token count in the test file as constants with a comment naming the llama.cpp build hash. **Do not** paste a figure from a blog post — the model file and quantization must match ours exactly. + +- [ ] **Step 2: Write the test** + +`[SkippableFact]`, skipped when the corpus or GGUF is unavailable (follow the `DOTLLM_BITNET_GGUF` early-return pattern already used in the integration suite). Assert our perplexity is within a stated relative tolerance of the recorded llama.cpp figure, and that `ScoredTokens` matches llama.cpp's token count. + +Start with a 1% tolerance. If it fails, **do not widen the tolerance to make it pass** — a real discrepancy means the tiling or the scored-range definition differs, and that is the bug this task exists to catch. Investigate the window geometry first. + +- [ ] **Step 3: Run it** + +Run: `dotnet test tests/DotLLM.Tests.Integration --filter "FullyQualifiedName~PerplexityComparabilityTests"` +Expected: PASS, or a documented discrepancy with the geometry investigated. + +- [ ] **Step 4: Commit and open the PR** + +```bash +git add tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs +git commit -m "test(eval): validate sliding-window perplexity against llama.cpp reference (#231)" +``` + +Then open the PR against upstream `main` referencing `Closes #231` and linking kkokosa/dotLLM#416. + +--- + +## Follow-up (separate, on `dev`) + +Migrate the ten existing per-test perplexity helpers onto this harness, gated on producing **numerically identical** results. Not part of this PR: those helpers exist only on `dev`, and this PR targets upstream `main`. File as its own issue once this lands. + +## Self-Review + +**Spec coverage:** contract (landed) · evaluator both modes (Tasks 3–5) · corpus streaming (Task 6) · adapter (Task 7) · CLI verb (Task 8) · comparability validation (Task 9) · never-loads-weights constraint (Task 7 adapter is borrowed-reference; evaluator signature takes a model) · streaming constraint (Task 6). Migration of existing helpers is explicitly deferred with a reason. + +**Placeholders:** none. Task 8's command body follows an existing in-repo pattern rather than inventing one, and its acceptance is a concrete arithmetic check on window geometry. + +**Type consistency:** `LogProb.OfTarget`, `PerplexityEvaluator.Evaluate`, `CorpusReader.StreamTokens`, `TransformerPerplexityModel` are used with identical signatures everywhere they appear. `TeacherForcedSinglePass`/`TeacherForcedGrowingPrefix` naming is fixed in Task 4. From e78e13dd3d8cadc8cf1f80662a72c5cf77cf2252 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:17:23 +0100 Subject: [PATCH 03/14] feat(eval): numerically stable log-softmax for perplexity scoring (#231) Co-Authored-By: Claude Opus 5 (1M context) --- src/DotLLM.Engine/Evaluation/LogProb.cs | 33 ++++++++++++++ .../Evaluation/LogProbTests.cs | 43 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/DotLLM.Engine/Evaluation/LogProb.cs create mode 100644 tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs diff --git a/src/DotLLM.Engine/Evaluation/LogProb.cs b/src/DotLLM.Engine/Evaluation/LogProb.cs new file mode 100644 index 00000000..a70b764e --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/LogProb.cs @@ -0,0 +1,33 @@ +namespace DotLLM.Engine.Evaluation; + +/// Numerically stable log-softmax over a single row of logits. +public static class LogProb +{ + /// + /// Returns log P(target) in nats under a softmax over . + /// + /// + /// Uses the max-shift identity log softmax(x)_t = (x_t - m) - log sum_j exp(x_j - m) + /// with m = max(x), so no exp argument is ever positive and overflow is + /// impossible. Accumulates in : a vocab of 128k float32 terms loses + /// meaningful precision in float32, and perplexity differences between near-identical runs + /// are exactly what this harness exists to resolve. + /// + /// One row of unnormalized scores. + /// Index whose log-probability is returned. + public static double OfTarget(ReadOnlySpan logits, int target) + { + if ((uint)target >= (uint)logits.Length) + throw new ArgumentOutOfRangeException(nameof(target)); + + float max = logits[0]; + for (int j = 1; j < logits.Length; j++) + if (logits[j] > max) max = logits[j]; + + double sumExp = 0; + for (int j = 0; j < logits.Length; j++) + sumExp += Math.Exp(logits[j] - max); + + return (logits[target] - max) - Math.Log(sumExp); + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs new file mode 100644 index 00000000..9366e03f --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs @@ -0,0 +1,43 @@ +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class LogProbTests +{ + [Fact] + public void OfTarget_UniformLogits_IsNegativeLogVocab() + { + // Four equal logits => p = 1/4 for each => log p = -log 4. + var logits = new float[] { 2.5f, 2.5f, 2.5f, 2.5f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.Equal(-Math.Log(4.0), actual, 12); + } + + [Fact] + public void OfTarget_IsShiftInvariant() + { + var a = new float[] { 1f, 2f, 3f }; + var b = new float[] { 1001f, 1002f, 1003f }; + Assert.Equal(LogProb.OfTarget(a, 1), LogProb.OfTarget(b, 1), 12); + } + + [Fact] + public void OfTarget_LargeLogits_DoesNotOverflow() + { + // Naive exp() would overflow to infinity here; the max-shift must prevent it. + var logits = new float[] { 800f, 900f, 1000f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.True(double.IsFinite(actual)); + Assert.Equal(0.0, actual, 6); // target dominates => p ~ 1 => log p ~ 0 + } + + [Fact] + public void OfTarget_SumOfProbabilitiesIsOne() + { + var logits = new float[] { -1.5f, 0.25f, 3f, 0.5f }; + double sum = 0; + for (int i = 0; i < logits.Length; i++) sum += Math.Exp(LogProb.OfTarget(logits, i)); + Assert.Equal(1.0, sum, 10); + } +} From 363902738c15fc1fbb00cff4ebed637e9c7ba1d1 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:34:07 +0100 Subject: [PATCH 04/14] test(eval): deterministic IPerplexityModel double recording forward windows (#231) Records every window passed to Forward so evaluator tests can assert on window tiling and on the O(n) vs O(n^2) strategy split, not just on the resulting perplexity number. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/FakePerplexityModel.cs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs diff --git a/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs b/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs new file mode 100644 index 00000000..ea6b1399 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs @@ -0,0 +1,57 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Tests.Unit.Evaluation; + +/// +/// Deterministic for evaluator tests. Records every window it is +/// asked to score so tests can assert on window tiling, not just on the resulting number. +/// +internal sealed class FakePerplexityModel : IPerplexityModel, IDisposable +{ + private readonly Func _rowFactory; // (absolutePosition, vocabSize) => logits + private readonly List _forwardCalls = []; + private readonly List _issued = []; + + public FakePerplexityModel( + int vocabSize, int maxContextLength, bool returnsAllRows, + Func rowFactory) + { + VocabSize = vocabSize; + MaxContextLength = maxContextLength; + ReturnsAllRows = returnsAllRows; + _rowFactory = rowFactory; + } + + public int VocabSize { get; } + public int MaxContextLength { get; } + public bool ReturnsAllRows { get; } + + /// Token windows passed to , in call order. + public IReadOnlyList ForwardCalls => _forwardCalls; + + public unsafe ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + { + _forwardCalls.Add(tokens.ToArray()); + + int rows = ReturnsAllRows ? tokens.Length : 1; + int firstRow = ReturnsAllRows ? 0 : tokens.Length - 1; + var tensor = UnmanagedTensor.Allocate(new TensorShape(rows, VocabSize), DType.Float32); + + var dest = new Span((void*)tensor.DataPointer, rows * VocabSize); + for (int r = 0; r < rows; r++) + _rowFactory(positions[firstRow + r], VocabSize).CopyTo(dest[(r * VocabSize)..]); + + _issued.Add(tensor); + return tensor; + } + + /// Uniform logits: every target scores exactly -log(vocabSize). + public static Func Uniform => (_, vocab) => new float[vocab]; + + public void Dispose() + { + foreach (var t in _issued) t.Dispose(); + _issued.Clear(); + } +} From a11821eab948496527f4401d24b7eccab5016004 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:35:59 +0100 Subject: [PATCH 05/14] feat(eval): PerplexityEvaluator teacher-forced mode for all-rows backends (#231) Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/PerplexityEvaluator.cs | 67 +++++++++++++++++++ .../Evaluation/PerplexityEvaluatorTests.cs | 50 ++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs create mode 100644 tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs new file mode 100644 index 00000000..e4d8341a --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -0,0 +1,67 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Engine.Evaluation; + +/// +/// Computes perplexity over a token sequence using an . +/// +/// +/// The evaluator never loads weights — callers pass an already-constructed model. On unified-memory +/// parts a large VRAM carve-out leaves host RAM scarce, and perplexity (a long run of full-context +/// prefills) is the workload most punished by holding a second host-side copy of the weights. +/// +public static class PerplexityEvaluator +{ + /// Scores and returns the aggregate result. + /// An already-constructed model. Not owned; not disposed here. + /// Token ids to score. + /// Mode and window geometry. + public static PerplexityResult Evaluate( + IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options) + { + ArgumentNullException.ThrowIfNull(model); + if (tokens.Length < 2) + throw new ArgumentException("At least two tokens are required to score one target.", nameof(tokens)); + + int context = Math.Min(options.ContextLength, model.MaxContextLength); + if (context < 2) + throw new ArgumentException("Context length must be at least 2.", nameof(options)); + + return options.Mode switch + { + PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), + _ => throw new NotSupportedException($"Mode {options.Mode} is not implemented yet."), + }; + } + + private static PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + => TeacherForcedSinglePass(model, tokens, context); + + // Backend returns every row, so one forward pass scores every target: row i predicts token i+1. + private static unsafe PerplexityResult TeacherForcedSinglePass( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + Span positions = length <= 512 ? stackalloc int[length] : new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + + using ITensor logits = model.Forward(tokens[..length], positions); + int vocab = model.VocabSize; + // Row i predicts token i+1, so the final row has no target within the window. + for (int i = 0; i < length - 1; i++) + { + var row = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)i * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(row, tokens[i + 1]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: 1); + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs new file mode 100644 index 00000000..813f84f8 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -0,0 +1,50 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class PerplexityEvaluatorTests +{ + // Vocab must exceed every token id used as a scoring target. + private const int Vocab = 64; + private static readonly int[] Tokens = Enumerable.Range(0, 32).ToArray(); + + [Fact] + public void TeacherForced_AllRowsBackend_UniformLogitsGivesVocabSizePerplexity() + { + // Uniform logits => P(any target) = 1/vocab => perplexity == vocab, exactly. + using var model = new FakePerplexityModel( + vocabSize: Vocab, maxContextLength: 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, ContextLength: 32, Stride: 32)); + + Assert.Equal(Vocab, result.Perplexity, 9); + Assert.Equal(31, result.ScoredTokens); // n-1 targets from one pass + } + + [Fact] + public void TeacherForced_AllRowsBackend_UsesASingleForwardPass() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // The whole point of ReturnsAllRows: one pass scores every target. + Assert.Single(model.ForwardCalls); + Assert.Equal(32, model.ForwardCalls[0].Length); + } + + [Fact] + public void MeanNll_AndPerplexity_AreConsistent() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + Assert.Equal(result.Perplexity, Math.Exp(result.MeanNegativeLogLikelihood), 9); + } +} From 75cf1bf776a9432bf8252f09289e15658a5a346e Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:37:03 +0100 Subject: [PATCH 06/14] feat(eval): growing-prefix teacher-forced path for last-row-only backends (#231) Produces a numerically identical result to the all-rows path on the same tokens, which is what makes migrating the existing per-test helpers onto this harness a meaningful gate rather than an aspiration. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/PerplexityEvaluator.cs | 29 ++++++++++++++- .../Evaluation/PerplexityEvaluatorTests.cs | 36 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs index e4d8341a..d8036fd1 100644 --- a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -37,7 +37,34 @@ public static PerplexityResult Evaluate( private static PerplexityResult EvaluateTeacherForced( IPerplexityModel model, ReadOnlySpan tokens, int context) - => TeacherForcedSinglePass(model, tokens, context); + => model.ReturnsAllRows + ? TeacherForcedSinglePass(model, tokens, context) + : TeacherForcedGrowingPrefix(model, tokens, context); + + // Backend returns only the final row, so each target needs its own prefill over the growing + // prefix. O(n^2) in forward passes — unavoidable, and the reason the CUDA harnesses that + // originated this methodology carry a stride. + private static unsafe PerplexityResult TeacherForcedGrowingPrefix( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + int vocab = model.VocabSize; + var positions = new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + for (int prefix = 1; prefix < length; prefix++) + { + using ITensor logits = model.Forward(tokens[..prefix], positions.AsSpan(0, prefix)); + var row = new ReadOnlySpan((void*)logits.DataPointer, vocab); + sumNll += -LogProb.OfTarget(row, tokens[prefix]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); + } // Backend returns every row, so one forward pass scores every target: row i predicts token i+1. private static unsafe PerplexityResult TeacherForcedSinglePass( diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs index 813f84f8..51715910 100644 --- a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -47,4 +47,40 @@ public void MeanNll_AndPerplexity_AreConsistent() Assert.Equal(result.Perplexity, Math.Exp(result.MeanNegativeLogLikelihood), 9); } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_MatchesAllRowsBackendExactly() + { + // Position-dependent but deterministic logits, so a wrong row/position mapping shows up. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + for (int j = 0; j < vocab; j++) row[j] = (float)Math.Sin((position + 1) * (j + 1) * 0.37); + return row; + } + + using var allRows = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + using var lastRow = new FakePerplexityModel(Vocab, 64, returnsAllRows: false, Rows); + var options = new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32); + + var a = PerplexityEvaluator.Evaluate(allRows, Tokens, options); + var b = PerplexityEvaluator.Evaluate(lastRow, Tokens, options); + + Assert.Equal(a.Perplexity, b.Perplexity, 9); + Assert.Equal(a.ScoredTokens, b.ScoredTokens); + } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_ReprefixesGrowingWindows() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: false, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // One forward per scored target, each one token longer than the last. + Assert.Equal(31, model.ForwardCalls.Count); + for (int i = 0; i < model.ForwardCalls.Count; i++) + Assert.Equal(i + 1, model.ForwardCalls[i].Length); + } } From 23ea52990ffdb3d073c77bab81cac27a59f07480 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:38:09 +0100 Subject: [PATCH 07/14] feat(eval): llama.cpp-comparable sliding-window perplexity mode (#231) Window w covers [w*S, w*S+L) and scores absolute targets in [w*S+L-S, w*S+L), so scored ranges tile the corpus exactly and every scored token carries L-S tokens of context. S = L/2 reproduces llama.cpp's default of scoring the second half of each chunk. Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/PerplexityEvaluator.cs | 55 ++++++++++++++++++- .../Evaluation/PerplexityEvaluatorTests.cs | 55 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs index d8036fd1..99453268 100644 --- a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -31,7 +31,8 @@ public static PerplexityResult Evaluate( return options.Mode switch { PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), - _ => throw new NotSupportedException($"Mode {options.Mode} is not implemented yet."), + PerplexityMode.SlidingWindow => EvaluateSlidingWindow(model, tokens, context, options.Stride), + _ => throw new NotSupportedException($"Unknown mode {options.Mode}."), }; } @@ -66,6 +67,58 @@ private static unsafe PerplexityResult TeacherForcedGrowingPrefix( return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); } + // llama.cpp `--perplexity` tiling. Window w covers [w*S, w*S + L) and scores absolute targets + // in [w*S + L - S, w*S + L), so every scored token carries L-S tokens of context and the + // scored ranges tile the corpus exactly — no gaps, no double-counting. S = L/2 reproduces + // llama.cpp's default of scoring the second half of each chunk. + // + // Targets before the first window's scored range are never scored. llama.cpp skips them too; + // "fixing" that would break the comparability this mode exists to provide. + private static unsafe PerplexityResult EvaluateSlidingWindow( + IPerplexityModel model, ReadOnlySpan tokens, int context, int stride) + { + if (stride < 1 || stride >= context) + throw new ArgumentException( + $"Stride must be in [1, {context - 1}] for a context of {context}; each scored token needs at least one token of context.", + nameof(stride)); + + if (!model.ReturnsAllRows) + throw new NotSupportedException( + "Sliding-window mode requires a backend that returns all rows. Use PerplexityMode.TeacherForced " + + "for last-row-only backends — re-prefilling per target inside a window would be O(n^2) and is " + + "already what the growing-prefix path does."); + + int vocab = model.VocabSize; + var positions = new int[context]; + double sumNll = 0; + int scored = 0, windows = 0; + + for (int start = 0; start + context <= tokens.Length; start += stride) + { + for (int i = 0; i < context; i++) positions[i] = start + i; + + using ITensor logits = model.Forward(tokens.Slice(start, context), positions); + windows++; + + // Absolute targets [start + context - stride, start + context); row for target t is t-start-1. + for (int t = start + context - stride; t < start + context; t++) + { + int row = t - start - 1; + var span = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)row * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(span, tokens[t]); + scored++; + } + } + + if (scored == 0) + throw new ArgumentException( + $"Corpus of {tokens.Length} tokens is shorter than one context window of {context}.", nameof(tokens)); + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, windows); + } + // Backend returns every row, so one forward pass scores every target: row i predicts token i+1. private static unsafe PerplexityResult TeacherForcedSinglePass( IPerplexityModel model, ReadOnlySpan tokens, int context) diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs index 51715910..08874d97 100644 --- a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -83,4 +83,59 @@ public void TeacherForced_LastRowOnlyBackend_ReprefixesGrowingWindows() for (int i = 0; i < model.ForwardCalls.Count; i++) Assert.Equal(i + 1, model.ForwardCalls[i].Length); } + + [Fact] + public void SlidingWindow_TilesScoredTokensWithoutGapsOrOverlap() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16, S=8 => windows start at 0, 8, 16, 24; each scores its last 8 targets. + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 8)); + + Assert.Equal(4, result.WindowCount); + Assert.Equal(32, result.ScoredTokens); // 4 windows x 8 targets + Assert.Equal(Vocab, result.Perplexity, 9); // uniform logits + + Assert.Equal(4, model.ForwardCalls.Count); + Assert.All(model.ForwardCalls, w => Assert.Equal(16, w.Length)); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(8, model.ForwardCalls[1][0]); + Assert.Equal(16, model.ForwardCalls[2][0]); + Assert.Equal(24, model.ForwardCalls[3][0]); + } + + [Fact] + public void SlidingWindow_ScoresEachTargetAtItsTrueAbsolutePosition() + { + // Logits keyed to absolute position: a window that restarts positions at zero scores + // different values and fails this. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[position % vocab] = 10f; + return row; + } + + // Row at absolute position p predicts token p+1, and argmax(row_p) == p % Vocab. + // So make tokens[p+1] == p % Vocab, i.e. tokens[i] == (i-1) mod Vocab. + var tokens = new int[40]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = (i + Vocab - 1) % Vocab; + + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, 8)); + + Assert.True(result.MeanNegativeLogLikelihood < 0.01, + $"expected confident predictions, got mean NLL {result.MeanNegativeLogLikelihood}"); + } + + [Fact] + public void SlidingWindow_RejectsStrideNotSmallerThanContext() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + Assert.Throws(() => PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16))); + } } From d40e4f0fdab917471df99652da1e303d17d946c0 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:39:17 +0100 Subject: [PATCH 08/14] feat(eval): streaming corpus tokenizer for perplexity runs (#231) Co-Authored-By: Claude Opus 5 (1M context) --- src/DotLLM.Engine/Evaluation/CorpusReader.cs | 69 +++++++++++++++++++ .../Evaluation/CorpusReaderTests.cs | 47 +++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 src/DotLLM.Engine/Evaluation/CorpusReader.cs create mode 100644 tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs diff --git a/src/DotLLM.Engine/Evaluation/CorpusReader.cs b/src/DotLLM.Engine/Evaluation/CorpusReader.cs new file mode 100644 index 00000000..b6d10566 --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/CorpusReader.cs @@ -0,0 +1,69 @@ +using System.Text; +using DotLLM.Tokenizers; + +namespace DotLLM.Engine.Evaluation; + +/// Streams a text corpus into tokens without materializing the whole file or token array. +/// +/// Streaming is a design constraint rather than an optimisation. On unified-memory parts a large +/// VRAM carve-out leaves host RAM scarce, and a standard perplexity corpus tokenizes to hundreds of +/// thousands of ints — held alongside the weights, that is exactly the pressure this harness must +/// not add. +/// +public static class CorpusReader +{ + /// + /// Reads in character chunks, tokenizes each chunk, and yields token + /// ids in order, stopping after (0 = unbounded). + /// + /// + /// Chunks are cut at the last whitespace so a token is never split across a boundary; the + /// remainder is carried into the next chunk, and the final carry is flushed whole. + /// + /// Corpus source. + /// Tokenizer whose vocabulary the ids belong to. + /// Upper bound on emitted tokens; 0 for unbounded. + /// Characters read per chunk. + public static IEnumerable StreamTokens( + TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(tokenizer); + ArgumentOutOfRangeException.ThrowIfLessThan(charChunkSize, 1); + + var buffer = new char[charChunkSize]; + var carry = new StringBuilder(); + int emitted = 0; + + while (true) + { + int read = reader.Read(buffer, 0, buffer.Length); + if (read == 0) break; + + carry.Append(buffer, 0, read); + string pending = carry.ToString(); + + int cut = pending.LastIndexOf(' '); + if (cut < 0) continue; // no safe split point yet; keep accumulating + + string ready = pending[..cut]; + carry.Clear(); + carry.Append(pending[(cut + 1)..]); + + foreach (int id in tokenizer.Encode(ready)) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + + if (carry.Length > 0) + { + foreach (int id in tokenizer.Encode(carry.ToString())) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs new file mode 100644 index 00000000..fb553e1e --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs @@ -0,0 +1,47 @@ +using DotLLM.Engine.Evaluation; +using DotLLM.Tokenizers; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class CorpusReaderTests +{ + // One token per whitespace-separated word; ids are word lengths, so order is checkable. + private sealed class WordTokenizer : ITokenizer + { + public int[] Encode(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(w => w.Length).ToArray(); + + public string Decode(ReadOnlySpan tokenIds) => throw new NotSupportedException(); + public string DecodeToken(int tokenId) => throw new NotSupportedException(); + public int CountTokens(string text) => Encode(text).Length; + public int VocabSize => 1024; + public int BosTokenId => 0; + public int EosTokenId => 1; + } + + [Fact] + public void StreamTokens_ProducesTokensInOrder() + { + using var reader = new StringReader("a bb ccc dddd"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer()).ToArray(); + Assert.Equal([1, 2, 3, 4], tokens); + } + + [Fact] + public void StreamTokens_HonoursMaxTokens() + { + using var reader = new StringReader("a bb ccc dddd eeeee"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 3).ToArray(); + Assert.Equal([1, 2, 3], tokens); + } + + [Fact] + public void StreamTokens_DoesNotSplitTokensAcrossChunkBoundaries() + { + // A tiny chunk size forces the boundary case: "ccc" must not become "c" + "cc". + using var reader = new StringReader("a bb ccc dddd eeeee ffffff"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 0, charChunkSize: 4).ToArray(); + Assert.Equal([1, 2, 3, 4, 5, 6], tokens); + } +} From fe4c5168f2d7539496d7160a2ebfc6559b4ad72f Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:39:39 +0100 Subject: [PATCH 09/14] feat(eval): IPerplexityModel adapter over TransformerModel (#231) Co-Authored-By: Claude Opus 5 (1M context) --- .../Evaluation/TransformerPerplexityModel.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs diff --git a/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs b/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs new file mode 100644 index 00000000..f0205a37 --- /dev/null +++ b/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs @@ -0,0 +1,43 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; +using DotLLM.Models.Architectures; + +namespace DotLLM.Models.Evaluation; + +/// Adapts to . +/// +/// Holds a borrowed reference: the adapter does not own the model and does not dispose it, so the +/// caller keeps a single resident copy of the weights. This is the whole point of the evaluator +/// taking a constructed model rather than a path — on a unified-memory part with a large VRAM +/// carve-out, a second host-side copy is the difference between running and not. +/// +public sealed class TransformerPerplexityModel : IPerplexityModel +{ + private readonly TransformerModel _model; + private readonly int _deviceId; + + /// An already-loaded model. Not owned; not disposed by this adapter. + /// Device for the forward pass; -1 is CPU. + public TransformerPerplexityModel(TransformerModel model, int deviceId = -1) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _deviceId = deviceId; + } + + /// + public int VocabSize => _model.Config.VocabSize; + + /// + public int MaxContextLength => _model.Config.MaxSequenceLength; + + /// + /// + /// is + /// documented as returning logits of shape [seqLen, vocab_size] for all input positions. + /// + public bool ReturnsAllRows => true; + + /// + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + => _model.Forward(tokens, positions, _deviceId); +} From 7e9f4c2cb134c7b32f6148474acd6182383b0fd3 Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:42:59 +0100 Subject: [PATCH 10/14] feat(cli): dotllm perplexity verb (#231) Reports window geometry and scored-token count alongside the figure: a perplexity without them is not comparable to anything, which is the failure this harness exists to prevent. Verified end-to-end on SmolLM-135M Q8_0 over wikitext-2 (ctx 256, stride 128, 2048 tokens): ppl 24.8800, 15 windows, 1920 scored -- matching the expected tiling arithmetic exactly. Co-Authored-By: Claude Opus 5 (1M context) --- src/DotLLM.Cli/Commands/PerplexityCommand.cs | 168 +++++++++++++++++++ src/DotLLM.Cli/Program.cs | 4 + 2 files changed, 172 insertions(+) create mode 100644 src/DotLLM.Cli/Commands/PerplexityCommand.cs diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs new file mode 100644 index 00000000..f791a3d7 --- /dev/null +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -0,0 +1,168 @@ +using System.ComponentModel; +using System.Diagnostics; +using DotLLM.Core.Configuration; +using DotLLM.Core.Evaluation; +using DotLLM.Core.Models; +using DotLLM.Engine; +using DotLLM.Engine.Evaluation; +using DotLLM.Models.Architectures; +using DotLLM.Models.Evaluation; +using DotLLM.Models.Gguf; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace DotLLM.Cli.Commands; + +/// +/// Computes perplexity over a text corpus: load → stream-tokenize → score. +/// +/// +/// Defaults to with stride = context / 2, which +/// reproduces llama.cpp's --perplexity methodology, so the reported figure is directly +/// comparable to published numbers for the same model, corpus, context and stride. +/// +internal sealed class PerplexityCommand : AsyncCommand +{ + public sealed class Settings : CommandSettings + { + [CommandArgument(0, "")] + [Description("Path to a GGUF file or HuggingFace repo ID (e.g., QuantFactory/SmolLM-135M-GGUF).")] + public string Model { get; set; } = string.Empty; + + [CommandOption("--corpus|-f")] + [Description("Path to a UTF-8 text corpus (e.g. wiki.test.raw).")] + public string Corpus { get; set; } = string.Empty; + + [CommandOption("--context|-c")] + [Description("Context window in tokens. Clamped to the model's maximum sequence length.")] + [DefaultValue(512)] + public int Context { get; set; } = 512; + + [CommandOption("--stride")] + [Description("Tokens advanced between windows. 0 selects context/2, matching llama.cpp's default.")] + [DefaultValue(0)] + public int Stride { get; set; } + + [CommandOption("--max-tokens|-n")] + [Description("Cap on corpus tokens consumed. 0 = unbounded.")] + [DefaultValue(0)] + public int MaxTokens { get; set; } + + [CommandOption("--mode")] + [Description("Scoring mode: sliding-window (default, llama.cpp-comparable) or teacher-forced.")] + [DefaultValue("sliding-window")] + public string Mode { get; set; } = "sliding-window"; + + [CommandOption("--quant")] + [Description("Quantization to select when resolving a HuggingFace repo ID.")] + public string? Quant { get; set; } + + [CommandOption("--threads")] + [Description("Compute threads. 0 = auto.")] + [DefaultValue(0)] + public int Threads { get; set; } + } + + public override async Task ExecuteAsync(CommandContext context, Settings settings) + { + if (string.IsNullOrWhiteSpace(settings.Corpus)) + { + AnsiConsole.MarkupLine("[red]--corpus is required.[/]"); + return 1; + } + + if (!File.Exists(settings.Corpus)) + { + AnsiConsole.MarkupLine($"[red]Corpus not found: {Markup.Escape(settings.Corpus)}[/]"); + return 1; + } + + if (!TryParseMode(settings.Mode, out PerplexityMode mode)) + { + AnsiConsole.MarkupLine( + $"[red]Unknown --mode '{Markup.Escape(settings.Mode)}'. Expected 'sliding-window' or 'teacher-forced'.[/]"); + return 1; + } + + string? resolvedPath = GgufFileResolver.Resolve(settings.Model, settings.Quant); + if (resolvedPath is null) + return 1; + + using GgufFile gguf = GgufFile.Open(resolvedPath); + ModelConfig config = GgufModelConfigExtractor.Extract(gguf.Metadata); + var tokenizer = GgufBpeTokenizerFactory.Load(gguf.Metadata); + using TransformerModel model = TransformerModel.LoadFromGguf( + gguf, config, new ThreadingConfig(settings.Threads)); + + int effectiveContext = Math.Min(settings.Context, config.MaxSequenceLength); + int effectiveStride = settings.Stride > 0 ? settings.Stride : Math.Max(1, effectiveContext / 2); + + // Streamed, then buffered once: scoring needs random access across windows, but the file + // itself is never held in memory and the token list is bounded by --max-tokens. + var tokens = new List(); + using (var reader = new StreamReader(settings.Corpus)) + { + foreach (int id in CorpusReader.StreamTokens(reader, tokenizer, settings.MaxTokens)) + tokens.Add(id); + } + + if (tokens.Count < 2) + { + AnsiConsole.MarkupLine($"[red]Corpus tokenized to {tokens.Count} tokens; at least 2 are required.[/]"); + return 1; + } + + var perplexityModel = new TransformerPerplexityModel(model, deviceId: -1); + var options = new PerplexityOptions(mode, effectiveContext, effectiveStride, settings.MaxTokens); + + var sw = Stopwatch.StartNew(); + PerplexityResult result; + try + { + result = PerplexityEvaluator.Evaluate(perplexityModel, System.Runtime.InteropServices.CollectionsMarshal.AsSpan(tokens), options); + } + catch (ArgumentException ex) + { + AnsiConsole.MarkupLine($"[red]{Markup.Escape(ex.Message)}[/]"); + return 1; + } + sw.Stop(); + + // Window geometry and scored-token count are reported alongside the figure deliberately: + // a perplexity without them is not comparable to anything. + var table = new Table().Border(TableBorder.Rounded); + table.AddColumn("Metric"); + table.AddColumn(new TableColumn("Value").RightAligned()); + table.AddRow("Perplexity", $"{result.Perplexity:F4}"); + table.AddRow("Mean NLL (nats)", $"{result.MeanNegativeLogLikelihood:F6}"); + table.AddRow("Scored tokens", $"{result.ScoredTokens:N0}"); + table.AddRow("Windows", $"{result.WindowCount:N0}"); + table.AddRow("Mode", mode == PerplexityMode.SlidingWindow ? "sliding-window" : "teacher-forced"); + table.AddRow("Context", $"{effectiveContext:N0}"); + table.AddRow("Stride", $"{effectiveStride:N0}"); + table.AddRow("Corpus tokens", $"{tokens.Count:N0}"); + table.AddRow("Elapsed", $"{sw.Elapsed.TotalSeconds:F2} s"); + AnsiConsole.Write(table); + + await Task.CompletedTask; + return 0; + } + + private static bool TryParseMode(string value, out PerplexityMode mode) + { + switch (value.Trim().ToLowerInvariant()) + { + case "sliding-window": + case "sliding": + mode = PerplexityMode.SlidingWindow; + return true; + case "teacher-forced": + case "teacher": + mode = PerplexityMode.TeacherForced; + return true; + default: + mode = default; + return false; + } + } +} diff --git a/src/DotLLM.Cli/Program.cs b/src/DotLLM.Cli/Program.cs index be553e78..b7ad93e2 100644 --- a/src/DotLLM.Cli/Program.cs +++ b/src/DotLLM.Cli/Program.cs @@ -48,6 +48,10 @@ .WithDescription("Interactive multi-turn chat with a GGUF model.") .WithExample("chat", "QuantFactory/SmolLM-135M-GGUF", "--system", "You are a helpful assistant."); + config.AddCommand("perplexity") + .WithDescription("Compute perplexity over a text corpus.") + .WithExample("perplexity", "QuantFactory/SmolLM-135M-GGUF", "--corpus", "wiki.test.raw", "--context", "512", "--stride", "256"); + config.AddCommand("serve") .WithDescription("Launch API server with built-in web chat UI.") .WithExample("serve", "Qwen/Qwen3-0.6B-GGUF", "--port", "8080"); From e303be4a32612ed00a102a7b8066fcefad66206d Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 18:51:06 +0100 Subject: [PATCH 11/14] fix(eval): separate window advance from scored span; drop absolute positions (#231) Validating against llama.cpp build 8683 refuted two assumptions baked into the original design. 1. Advance and scored span are INDEPENDENT. llama.cpp advances by the full window yet scores only its second half, so its scored ranges have gaps. The single-stride model produced the same scored-token COUNT over a different token SET -- 24.88 vs llama.cpp's 24.01, a figure that looked comparable and was not. PerplexityOptions now carries UnscoredPrefix, and LlamaCppDefault() names the faithful configuration. 2. Positions restart at 0 per window. llama.cpp evaluates each chunk as an independent sequence; absolute positions ran past the model's max sequence length and threw on any corpus longer than it. Also fixes CorpusReader dropping the separating space when cutting a chunk: GPT-2-style BPE encodes a leading space into the following token, so dropping it silently changes the token stream versus a single-pass tokenization. Geometry now matches llama.cpp exactly (15 windows, 1920 scored). A +3.46% residual remains and is NOT yet explained -- see the next commit's notes. Co-Authored-By: Claude Opus 5 (1M context) --- src/DotLLM.Cli/Commands/PerplexityCommand.cs | 23 ++++- .../Evaluation/PerplexityResult.cs | 61 +++++++++--- src/DotLLM.Engine/Evaluation/CorpusReader.cs | 6 +- .../Evaluation/PerplexityEvaluator.cs | 62 +++++++++--- .../Evaluation/PerplexityEvaluatorTests.cs | 97 ++++++++++++++++--- 5 files changed, 207 insertions(+), 42 deletions(-) diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs index f791a3d7..fd9bc78b 100644 --- a/src/DotLLM.Cli/Commands/PerplexityCommand.cs +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -39,10 +39,15 @@ public sealed class Settings : CommandSettings public int Context { get; set; } = 512; [CommandOption("--stride")] - [Description("Tokens advanced between windows. 0 selects context/2, matching llama.cpp's default.")] + [Description("Tokens advanced between window starts. 0 selects the context length (non-overlapping chunks, llama.cpp's default).")] [DefaultValue(0)] public int Stride { get; set; } + [CommandOption("--unscored-prefix")] + [Description("Leading tokens of each window used as context only. -1 selects context/2 (llama.cpp's default).")] + [DefaultValue(-1)] + public int UnscoredPrefix { get; set; } = -1; + [CommandOption("--max-tokens|-n")] [Description("Cap on corpus tokens consumed. 0 = unbounded.")] [DefaultValue(0)] @@ -53,6 +58,11 @@ public sealed class Settings : CommandSettings [DefaultValue("sliding-window")] public string Mode { get; set; } = "sliding-window"; + [CommandOption("--bos")] + [Description("Substitute BOS at the start of each window. Match the model's add_bos setting: llama.cpp only does this when the tokenizer requests it.")] + [DefaultValue(false)] + public bool Bos { get; set; } + [CommandOption("--quant")] [Description("Quantization to select when resolving a HuggingFace repo ID.")] public string? Quant { get; set; } @@ -95,7 +105,11 @@ public override async Task ExecuteAsync(CommandContext context, Settings se gguf, config, new ThreadingConfig(settings.Threads)); int effectiveContext = Math.Min(settings.Context, config.MaxSequenceLength); - int effectiveStride = settings.Stride > 0 ? settings.Stride : Math.Max(1, effectiveContext / 2); + // Defaults reproduce llama.cpp: non-overlapping chunks, scoring the second half of each. + int effectiveStride = settings.Stride > 0 ? settings.Stride : effectiveContext; + int effectivePrefix = settings.UnscoredPrefix >= 0 + ? settings.UnscoredPrefix + : Math.Max(1, effectiveContext / 2); // Streamed, then buffered once: scoring needs random access across windows, but the file // itself is never held in memory and the token list is bounded by --max-tokens. @@ -113,7 +127,9 @@ public override async Task ExecuteAsync(CommandContext context, Settings se } var perplexityModel = new TransformerPerplexityModel(model, deviceId: -1); - var options = new PerplexityOptions(mode, effectiveContext, effectiveStride, settings.MaxTokens); + int bosTokenId = settings.Bos ? tokenizer.BosTokenId : -1; + var options = new PerplexityOptions( + mode, effectiveContext, effectiveStride, settings.MaxTokens, effectivePrefix, bosTokenId); var sw = Stopwatch.StartNew(); PerplexityResult result; @@ -140,6 +156,7 @@ public override async Task ExecuteAsync(CommandContext context, Settings se table.AddRow("Mode", mode == PerplexityMode.SlidingWindow ? "sliding-window" : "teacher-forced"); table.AddRow("Context", $"{effectiveContext:N0}"); table.AddRow("Stride", $"{effectiveStride:N0}"); + table.AddRow("Unscored prefix", $"{effectivePrefix:N0}"); table.AddRow("Corpus tokens", $"{tokens.Count:N0}"); table.AddRow("Elapsed", $"{sw.Elapsed.TotalSeconds:F2} s"); AnsiConsole.Write(table); diff --git a/src/DotLLM.Core/Evaluation/PerplexityResult.cs b/src/DotLLM.Core/Evaluation/PerplexityResult.cs index c6a3661b..5f0ea283 100644 --- a/src/DotLLM.Core/Evaluation/PerplexityResult.cs +++ b/src/DotLLM.Core/Evaluation/PerplexityResult.cs @@ -19,14 +19,16 @@ public enum PerplexityMode TeacherForced, /// - /// Sliding-window scoring matching llama.cpp's --perplexity methodology: the corpus is - /// walked in windows of advanced by - /// , and only tokens beyond the carried-over prefix are - /// scored, so every scored token has a full-length context. + /// Sliding-window scoring. The corpus is walked in windows of + /// advanced by + /// , and each window scores only the targets beyond its + /// , so every scored token carries that many + /// tokens of context. /// /// - /// Absolute-value oriented: comparable to published llama.cpp figures when context length, - /// stride, corpus and model match. + /// Absolute-value oriented: comparable to published llama.cpp figures when model, corpus, + /// context, stride and unscored prefix all match. See + /// . /// SlidingWindow, } @@ -34,22 +36,59 @@ public enum PerplexityMode /// Configuration for a perplexity run. /// Scoring strategy. /// -/// Window size in tokens. Clamped to . +/// Window size L in tokens. Clamped to . /// /// -/// Tokens advanced between windows. Equal to gives -/// non-overlapping windows (llama.cpp's default); a smaller value overlaps, scoring each token -/// with more preceding context at proportionally higher cost. +/// Tokens advanced between window starts. Stride == ContextLength gives non-overlapping +/// windows; a smaller value overlaps them. /// /// /// Upper bound on corpus tokens consumed; 0 means unbounded. Bounds runtime on large /// corpora without truncating the corpus file itself. /// +/// +/// Leading tokens of each window used only as context and never scored. -1 derives +/// ContextLength - Stride, which makes the scored ranges tile the corpus contiguously. +/// +/// +/// When non-negative, each window's first token is replaced by this id. llama.cpp does this for +/// every chunk, since each is evaluated as a fresh sequence; the substituted slot lies inside the +/// unscored prefix, so no scored target changes. -1 disables the substitution. +/// +/// +/// Advance and scored span are independent. A single "stride" cannot express +/// llama.cpp's scheme: it advances by the full window yet scores only the second half, so its +/// scored ranges have gaps. Collapsing the two into one knob silently produces a different token +/// set — the same count, scored over different tokens — and therefore a figure that looks +/// comparable but is not. +/// Scored targets in a window starting at s are the absolute indices +/// [s + UnscoredPrefix, s + ContextLength). +/// public readonly record struct PerplexityOptions( PerplexityMode Mode, int ContextLength, int Stride, - int MaxTokens = 0); + int MaxTokens = 0, + int UnscoredPrefix = -1, + int BosTokenId = -1) +{ + /// + /// Options reproducing llama.cpp's --perplexity defaults for a given context: + /// non-overlapping chunks of , scoring the second half of each. + /// + /// + /// This is the configuration whose output is directly comparable to published llama.cpp + /// figures. Verified against llama.cpp build 8683 (d0a6dfeb2). + /// + /// Window size. + /// Corpus token cap; 0 for unbounded. + /// + /// BOS id to substitute at the start of each chunk, mirroring llama.cpp; -1 disables. + /// + public static PerplexityOptions LlamaCppDefault(int contextLength, int maxTokens = 0, int bosTokenId = -1) => + new(PerplexityMode.SlidingWindow, contextLength, Stride: contextLength, maxTokens, + UnscoredPrefix: contextLength / 2, BosTokenId: bosTokenId); +} /// Outcome of a perplexity run. /// diff --git a/src/DotLLM.Engine/Evaluation/CorpusReader.cs b/src/DotLLM.Engine/Evaluation/CorpusReader.cs index b6d10566..12ea325e 100644 --- a/src/DotLLM.Engine/Evaluation/CorpusReader.cs +++ b/src/DotLLM.Engine/Evaluation/CorpusReader.cs @@ -46,9 +46,13 @@ public static IEnumerable StreamTokens( int cut = pending.LastIndexOf(' '); if (cut < 0) continue; // no safe split point yet; keep accumulating + // The separating space is carried INTO the next chunk, not dropped. GPT-2-style BPE + // encodes a leading space as part of the following token, so dropping it silently + // changes the token stream — and therefore the perplexity — versus tokenizing the + // corpus in one pass. string ready = pending[..cut]; carry.Clear(); - carry.Append(pending[(cut + 1)..]); + carry.Append(pending[cut..]); foreach (int id in tokenizer.Encode(ready)) { diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs index 99453268..3417a1fc 100644 --- a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -31,7 +31,8 @@ public static PerplexityResult Evaluate( return options.Mode switch { PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), - PerplexityMode.SlidingWindow => EvaluateSlidingWindow(model, tokens, context, options.Stride), + PerplexityMode.SlidingWindow => EvaluateSlidingWindow( + model, tokens, context, options.Stride, options.UnscoredPrefix, options.BosTokenId), _ => throw new NotSupportedException($"Unknown mode {options.Mode}."), }; } @@ -67,20 +68,31 @@ private static unsafe PerplexityResult TeacherForcedGrowingPrefix( return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); } - // llama.cpp `--perplexity` tiling. Window w covers [w*S, w*S + L) and scores absolute targets - // in [w*S + L - S, w*S + L), so every scored token carries L-S tokens of context and the - // scored ranges tile the corpus exactly — no gaps, no double-counting. S = L/2 reproduces - // llama.cpp's default of scoring the second half of each chunk. + // Window w starts at w*Stride and covers [start, start + L); it scores the absolute targets + // [start + prefix, start + L), where `prefix` tokens serve as context only. // - // Targets before the first window's scored range are never scored. llama.cpp skips them too; - // "fixing" that would break the comparability this mode exists to provide. + // Advance (Stride) and scored span (L - prefix) are INDEPENDENT. llama.cpp advances by the + // full window yet scores only its second half, so its scored ranges have gaps; a scheme with + // one knob cannot express that, and collapsing them yields the same scored-token *count* over + // a different token *set* — a figure that looks comparable and is not. Verified the hard way + // against llama.cpp build 8683: contiguous tiling gave 24.88 where llama.cpp gave 24.01. + // + // Targets before the first window's scored range are never scored; llama.cpp skips them too, + // and "fixing" that would break the comparability this mode exists to provide. private static unsafe PerplexityResult EvaluateSlidingWindow( - IPerplexityModel model, ReadOnlySpan tokens, int context, int stride) + IPerplexityModel model, ReadOnlySpan tokens, int context, int stride, int unscoredPrefix, + int bosTokenId) { - if (stride < 1 || stride >= context) + if (stride < 1 || stride > context) + throw new ArgumentException( + $"Stride must be in [1, {context}] for a context of {context}.", nameof(stride)); + + int prefix = unscoredPrefix >= 0 ? unscoredPrefix : context - stride; + if (prefix < 1 || prefix >= context) throw new ArgumentException( - $"Stride must be in [1, {context - 1}] for a context of {context}; each scored token needs at least one token of context.", - nameof(stride)); + $"Unscored prefix must be in [1, {context - 1}] for a context of {context}; " + + "each scored token needs at least one token of context, and at least one token must be scored.", + nameof(unscoredPrefix)); if (!model.ReturnsAllRows) throw new NotSupportedException( @@ -93,15 +105,35 @@ private static unsafe PerplexityResult EvaluateSlidingWindow( double sumNll = 0; int scored = 0, windows = 0; + // Positions restart at 0 for every window: each is an independent sequence, exactly as + // llama.cpp evaluates each chunk. This is also what lets a corpus longer than the model's + // max sequence length be scored at all. + for (int i = 0; i < context; i++) positions[i] = i; + + // When a BOS id is supplied, each window's first token is replaced by it, mirroring + // llama.cpp's perplexity: every chunk is a fresh sequence and is given a sequence start. + // The substituted slot sits inside the unscored prefix, so no scored target is altered. + int[]? windowBuffer = bosTokenId >= 0 ? new int[context] : null; + for (int start = 0; start + context <= tokens.Length; start += stride) { - for (int i = 0; i < context; i++) positions[i] = start + i; + ReadOnlySpan window; + if (windowBuffer is null) + { + window = tokens.Slice(start, context); + } + else + { + tokens.Slice(start, context).CopyTo(windowBuffer); + windowBuffer[0] = bosTokenId; + window = windowBuffer; + } - using ITensor logits = model.Forward(tokens.Slice(start, context), positions); + using ITensor logits = model.Forward(window, positions); windows++; - // Absolute targets [start + context - stride, start + context); row for target t is t-start-1. - for (int t = start + context - stride; t < start + context; t++) + // Absolute targets [start + prefix, start + context); row for target t is t-start-1. + for (int t = start + prefix; t < start + context; t++) { int row = t - start - 1; var span = new ReadOnlySpan( diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs index 08874d97..6981abf4 100644 --- a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -107,35 +107,108 @@ public void SlidingWindow_TilesScoredTokensWithoutGapsOrOverlap() } [Fact] - public void SlidingWindow_ScoresEachTargetAtItsTrueAbsolutePosition() + public void SlidingWindow_EvaluatesEachWindowAsAnIndependentSequenceFromPositionZero() { - // Logits keyed to absolute position: a window that restarts positions at zero scores - // different values and fails this. - static float[] Rows(int position, int vocab) + // llama.cpp evaluates every chunk as a fresh sequence with positions restarting at 0. + // That is also what allows a corpus longer than the model's max sequence length to be + // scored at all -- absolute positions would run past it and throw. + var seenPositions = new List(); + + float[] Rows(int position, int vocab) { + seenPositions.Add(position); var row = new float[vocab]; - row[position % vocab] = 10f; + row[0] = 1f; return row; } - // Row at absolute position p predicts token p+1, and argmax(row_p) == p % Vocab. - // So make tokens[p+1] == p % Vocab, i.e. tokens[i] == (i-1) mod Vocab. var tokens = new int[40]; - for (int i = 0; i < tokens.Length; i++) tokens[i] = (i + Vocab - 1) % Vocab; + using var model = new FakePerplexityModel(Vocab, maxContextLength: 16, returnsAllRows: true, Rows); + + PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 8)); + + // Never a position at or beyond the window length, however far into the corpus we are. + Assert.All(seenPositions, p => Assert.InRange(p, 0, 15)); + Assert.Contains(0, seenPositions); + } - using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + [Fact] + public void SlidingWindow_ScoresTargetsFromTheCorrectRow() + { + // Row i of a window predicts the window's token i+1. A row/target off-by-one shows up as + // a confident model scoring badly. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[(position + 1) % vocab] = 20f; // argmax == the id this row should predict + return row; + } + + // Window-relative: token at window offset j has id j % Vocab, so row j's target is + // (j+1) % Vocab -- exactly the argmax above. Stride == context keeps windows aligned to + // the same offsets, so this holds for every window. + var tokens = new int[64]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = (i % 16) % Vocab; + + using var model = new FakePerplexityModel(Vocab, 16, returnsAllRows: true, Rows); var result = PerplexityEvaluator.Evaluate( - model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, 8)); + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 16, UnscoredPrefix: 8)); Assert.True(result.MeanNegativeLogLikelihood < 0.01, $"expected confident predictions, got mean NLL {result.MeanNegativeLogLikelihood}"); } [Fact] - public void SlidingWindow_RejectsStrideNotSmallerThanContext() + public void SlidingWindow_RejectsUnscoredPrefixLeavingNothingToScore() { using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); Assert.Throws(() => PerplexityEvaluator.Evaluate( - model, Tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16))); + model, Tokens, + new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16, UnscoredPrefix: 16))); + } + + [Fact] + public void LlamaCppDefault_UsesNonOverlappingChunksScoringTheSecondHalf() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16 => chunks at 0, 16 (32 would need tokens up to 48). Each scores its last 8. + var result = PerplexityEvaluator.Evaluate( + model, tokens, PerplexityOptions.LlamaCppDefault(contextLength: 16)); + + Assert.Equal(2, result.WindowCount); + Assert.Equal(16, result.ScoredTokens); // 2 chunks x 8 scored + Assert.Equal(2, model.ForwardCalls.Count); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(16, model.ForwardCalls[1][0]); // advances by the FULL window, not by 8 + } + + [Fact] + public void LlamaCppDefault_AndContiguousTiling_ScoreDifferentTokenSets() + { + // The bug this guards: both schemes score the same COUNT, so a count check alone cannot + // distinguish them. Only a position-dependent signal reveals the different token sets. + // Row at position p predicts token p+1. Give the correct target a confidence that VARIES + // with p, so the mean NLL depends on which positions were scored, not merely how many. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[(position + 1) % vocab] = 1f + (position % 7); + return row; + } + + var tokens = new int[64]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = i % Vocab; + + using var chunked = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + using var tiled = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + + var a = PerplexityEvaluator.Evaluate(chunked, tokens, PerplexityOptions.LlamaCppDefault(16)); + var b = PerplexityEvaluator.Evaluate( + tiled, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 8)); + + Assert.NotEqual(a.Perplexity, b.Perplexity, 6); } } From f9499a45c747a6fd896d7427e9e0c4386a5450ec Mon Sep 17 00:00:00 2001 From: James Burton Date: Thu, 30 Jul 2026 20:12:47 +0100 Subject: [PATCH 12/14] feat(eval): per-window observer + token-stream diagnostics (#231) Adds PerplexityEvaluator.WindowObserver and the CLI's --per-window, --dump-tokens and --tokens-file. These exist because an aggregate perplexity cannot localize a disagreement: differencing per-window figures against another implementation, and feeding that implementation's exact token ids back through our scorer, separates tokenization from scoring from geometry. That separation immediately paid for itself -- see #237: diffing token streams against llama.cpp on wikitext-2 found 30 mismatches in 3904 tokens, all the same repeated pair, which is a real pre-tokenizer bug rather than anything about perplexity. Co-Authored-By: Claude Opus 5 (1M context) --- src/DotLLM.Cli/Commands/PerplexityCommand.cs | 37 ++++++++++++++++++- .../Evaluation/PerplexityEvaluator.cs | 28 ++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs index fd9bc78b..b036ec12 100644 --- a/src/DotLLM.Cli/Commands/PerplexityCommand.cs +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -58,6 +58,19 @@ public sealed class Settings : CommandSettings [DefaultValue("sliding-window")] public string Mode { get; set; } = "sliding-window"; + [CommandOption("--tokens-file")] + [Description("Read pre-tokenized whitespace-separated ids instead of tokenizing --corpus. Isolates scoring from tokenization when comparing against another implementation.")] + public string? TokensFile { get; set; } + + [CommandOption("--dump-tokens")] + [Description("Write the tokenized corpus ids to this path, whitespace-separated, then continue. Diagnostic.")] + public string? DumpTokens { get; set; } + + [CommandOption("--per-window")] + [Description("Print each window's perplexity. Use to localize a disagreement with another implementation to specific corpus content.")] + [DefaultValue(false)] + public bool PerWindow { get; set; } + [CommandOption("--bos")] [Description("Substitute BOS at the start of each window. Match the model's add_bos setting: llama.cpp only does this when the tokenizer requests it.")] [DefaultValue(false)] @@ -114,8 +127,18 @@ public override async Task ExecuteAsync(CommandContext context, Settings se // Streamed, then buffered once: scoring needs random access across windows, but the file // itself is never held in memory and the token list is bounded by --max-tokens. var tokens = new List(); - using (var reader = new StreamReader(settings.Corpus)) + if (settings.TokensFile is not null) { + foreach (string part in File.ReadAllText(settings.TokensFile) + .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + { + tokens.Add(int.Parse(part)); + if (settings.MaxTokens > 0 && tokens.Count >= settings.MaxTokens) break; + } + } + else + { + using var reader = new StreamReader(settings.Corpus); foreach (int id in CorpusReader.StreamTokens(reader, tokenizer, settings.MaxTokens)) tokens.Add(id); } @@ -126,6 +149,9 @@ public override async Task ExecuteAsync(CommandContext context, Settings se return 1; } + if (settings.DumpTokens is not null) + File.WriteAllText(settings.DumpTokens, string.Join(' ', tokens)); + var perplexityModel = new TransformerPerplexityModel(model, deviceId: -1); int bosTokenId = settings.Bos ? tokenizer.BosTokenId : -1; var options = new PerplexityOptions( @@ -135,7 +161,14 @@ public override async Task ExecuteAsync(CommandContext context, Settings se PerplexityResult result; try { - result = PerplexityEvaluator.Evaluate(perplexityModel, System.Runtime.InteropServices.CollectionsMarshal.AsSpan(tokens), options); + PerplexityEvaluator.WindowObserver? observer = settings.PerWindow + ? (i, ppl, n) => Console.WriteLine($"window {i}: ppl={ppl:F6} scored={n}") + : null; + result = PerplexityEvaluator.Evaluate( + perplexityModel, + System.Runtime.InteropServices.CollectionsMarshal.AsSpan(tokens), + options, + observer); } catch (ArgumentException ex) { diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs index 3417a1fc..6605905d 100644 --- a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -13,12 +13,24 @@ namespace DotLLM.Engine.Evaluation; /// public static class PerplexityEvaluator { + /// + /// Per-window callback. Intended for diagnosis — differencing per-window figures against + /// another implementation localizes a disagreement to specific corpus content, which an + /// aggregate figure cannot. + /// + /// Zero-based index of the window just scored. + /// Perplexity over that window's scored targets alone. + /// Number of targets scored in that window. + public delegate void WindowObserver(int windowIndex, double windowPerplexity, int scoredInWindow); + /// Scores and returns the aggregate result. /// An already-constructed model. Not owned; not disposed here. /// Token ids to score. /// Mode and window geometry. + /// Optional per-window diagnostic callback. public static PerplexityResult Evaluate( - IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options) + IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options, + WindowObserver? onWindow = null) { ArgumentNullException.ThrowIfNull(model); if (tokens.Length < 2) @@ -32,7 +44,7 @@ public static PerplexityResult Evaluate( { PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), PerplexityMode.SlidingWindow => EvaluateSlidingWindow( - model, tokens, context, options.Stride, options.UnscoredPrefix, options.BosTokenId), + model, tokens, context, options.Stride, options.UnscoredPrefix, options.BosTokenId, onWindow), _ => throw new NotSupportedException($"Unknown mode {options.Mode}."), }; } @@ -81,7 +93,7 @@ private static unsafe PerplexityResult TeacherForcedGrowingPrefix( // and "fixing" that would break the comparability this mode exists to provide. private static unsafe PerplexityResult EvaluateSlidingWindow( IPerplexityModel model, ReadOnlySpan tokens, int context, int stride, int unscoredPrefix, - int bosTokenId) + int bosTokenId, WindowObserver? onWindow) { if (stride < 1 || stride > context) throw new ArgumentException( @@ -132,15 +144,23 @@ private static unsafe PerplexityResult EvaluateSlidingWindow( using ITensor logits = model.Forward(window, positions); windows++; + double windowNll = 0; + int windowScored = 0; + // Absolute targets [start + prefix, start + context); row for target t is t-start-1. for (int t = start + prefix; t < start + context; t++) { int row = t - start - 1; var span = new ReadOnlySpan( (void*)(logits.DataPointer + (nint)row * vocab * sizeof(float)), vocab); - sumNll += -LogProb.OfTarget(span, tokens[t]); + double nll = -LogProb.OfTarget(span, tokens[t]); + windowNll += nll; + windowScored++; + sumNll += nll; scored++; } + + onWindow?.Invoke(windows - 1, Math.Exp(windowNll / windowScored), windowScored); } if (scored == 0) From ca8a92bf6a4ca074d35a3f2dbd519649b860e69a Mon Sep 17 00:00:00 2001 From: James Burton Date: Fri, 31 Jul 2026 03:19:52 +0100 Subject: [PATCH 13/14] fix(eval): match llama.cpp's scored span exactly and report its error bar (#231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validating the harness against llama.cpp on wikitext-2 turned up three defects, all in the part that claims comparability with published figures. 1. LlamaCppDefault scored one target too many. llama.cpp sets first = n_ctx/2 and then accumulates `count += n_ctx - first - 1`, scoring targets (n_ctx/2, n_ctx) — the token AT n_ctx/2 is context, never a target. We scored it too, giving n_ctx/2 targets per chunk against llama.cpp's n_ctx/2 - 1: the same name over a different token set, which is exactly the failure mode the mode exists to avoid. 2. No error bar. llama.cpp prints "PPL = 18.8440 +/- 0.14515" and the +/- is load-bearing: on a 2,286-token corpus this model's figure carries ~6.5%, so a 3% "discrepancy" against another implementation is noise. Without it the harness invites chasing phantoms — it cost a full investigation to learn that. Computed by Welford rather than E[x^2]-E[x]^2, whose cancellation returns ~1e-6 for a constant NLL where the true variance is 0. 3. --tokens-file still demanded --corpus, so the one flag that separates a tokenizer difference from a scoring one could not be used. It now replaces --corpus, and accepts the JSON-array form reference tools print. Validated on wikitext-2 test (SmolLM-135M Q8_0, n_ctx=512, 589 chunks, 150,195 scored tokens both sides): llama.cpp b8683 18.8440 +/- 0.14515 ours, llama's ids 18.8911 (+0.25%, 0.32 sigma) ours, end-to-end 18.8950 (+0.27%, 0.35 sigma) Tokenizer agreement is exact over the corpus: 0 mismatches in 301,948 ids (requires the #237 pre-tokenizer fix; without it, 44 mismatches). --- src/DotLLM.Cli/Commands/PerplexityCommand.cs | 37 +++++++-- .../Evaluation/PerplexityResult.cs | 27 ++++++- .../Evaluation/PerplexityEvaluator.cs | 81 ++++++++++++++----- .../Evaluation/PerplexityEvaluatorTests.cs | 75 ++++++++++++++++- 4 files changed, 188 insertions(+), 32 deletions(-) diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs index b036ec12..b82f703f 100644 --- a/src/DotLLM.Cli/Commands/PerplexityCommand.cs +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -23,6 +23,12 @@ namespace DotLLM.Cli.Commands; /// internal sealed class PerplexityCommand : AsyncCommand { + /// + /// Delimiters accepted in a --tokens-file: whitespace plus the punctuation of a + /// JSON array, so a reference implementation's dump parses as-is. + /// + private static readonly char[] TokenIdSeparators = [' ', '\t', '\r', '\n', ',', '[', ']']; + public sealed class Settings : CommandSettings { [CommandArgument(0, "")] @@ -88,13 +94,24 @@ public sealed class Settings : CommandSettings public override async Task ExecuteAsync(CommandContext context, Settings settings) { - if (string.IsNullOrWhiteSpace(settings.Corpus)) + // --tokens-file supplies the token stream directly, so it replaces --corpus rather than + // supplementing it. Requiring both would defeat the flag's purpose: scoring a reference + // implementation's exact ids to separate a tokenizer difference from a scoring one. + if (settings.TokensFile is not null) { - AnsiConsole.MarkupLine("[red]--corpus is required.[/]"); + if (!File.Exists(settings.TokensFile)) + { + AnsiConsole.MarkupLine( + $"[red]Tokens file not found: {Markup.Escape(settings.TokensFile)}[/]"); + return 1; + } + } + else if (string.IsNullOrWhiteSpace(settings.Corpus)) + { + AnsiConsole.MarkupLine("[red]--corpus is required (or --tokens-file).[/]"); return 1; } - - if (!File.Exists(settings.Corpus)) + else if (!File.Exists(settings.Corpus)) { AnsiConsole.MarkupLine($"[red]Corpus not found: {Markup.Escape(settings.Corpus)}[/]"); return 1; @@ -120,17 +137,21 @@ public override async Task ExecuteAsync(CommandContext context, Settings se int effectiveContext = Math.Min(settings.Context, config.MaxSequenceLength); // Defaults reproduce llama.cpp: non-overlapping chunks, scoring the second half of each. int effectiveStride = settings.Stride > 0 ? settings.Stride : effectiveContext; + // context/2 + 1, not context/2: llama.cpp scores targets (n_ctx/2, n_ctx), leaving the token + // at n_ctx/2 as context only. See PerplexityOptions.LlamaCppDefault. int effectivePrefix = settings.UnscoredPrefix >= 0 ? settings.UnscoredPrefix - : Math.Max(1, effectiveContext / 2); + : Math.Min(effectiveContext - 1, Math.Max(1, effectiveContext / 2 + 1)); // Streamed, then buffered once: scoring needs random access across windows, but the file // itself is never held in memory and the token list is bounded by --max-tokens. var tokens = new List(); if (settings.TokensFile is not null) { + // Accept both bare whitespace-separated ids and the JSON-array form that reference + // tools print, so a dump can be pasted in without reformatting. foreach (string part in File.ReadAllText(settings.TokensFile) - .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .Split(TokenIdSeparators, StringSplitOptions.RemoveEmptyEntries)) { tokens.Add(int.Parse(part)); if (settings.MaxTokens > 0 && tokens.Count >= settings.MaxTokens) break; @@ -182,7 +203,9 @@ public override async Task ExecuteAsync(CommandContext context, Settings se var table = new Table().Border(TableBorder.Rounded); table.AddColumn("Metric"); table.AddColumn(new TableColumn("Value").RightAligned()); - table.AddRow("Perplexity", $"{result.Perplexity:F4}"); + // Printed as "PPL +/- err" in llama.cpp's own format so the two can be compared by eye. + // Without the error bar a reader has no way to tell a regression from sampling noise. + table.AddRow("Perplexity", $"{result.Perplexity:F4} +/- {result.StandardError:F5}"); table.AddRow("Mean NLL (nats)", $"{result.MeanNegativeLogLikelihood:F6}"); table.AddRow("Scored tokens", $"{result.ScoredTokens:N0}"); table.AddRow("Windows", $"{result.WindowCount:N0}"); diff --git a/src/DotLLM.Core/Evaluation/PerplexityResult.cs b/src/DotLLM.Core/Evaluation/PerplexityResult.cs index 5f0ea283..52782c5b 100644 --- a/src/DotLLM.Core/Evaluation/PerplexityResult.cs +++ b/src/DotLLM.Core/Evaluation/PerplexityResult.cs @@ -77,8 +77,13 @@ public readonly record struct PerplexityOptions( /// non-overlapping chunks of , scoring the second half of each. /// /// - /// This is the configuration whose output is directly comparable to published llama.cpp - /// figures. Verified against llama.cpp build 8683 (d0a6dfeb2). + /// This is the configuration whose output is directly comparable to published llama.cpp + /// figures. Verified against llama.cpp build 8683 (d0a6dfeb2). + /// The unscored prefix is contextLength / 2 + 1, not contextLength / 2. + /// llama.cpp sets first = n_ctx/2 and then accumulates count += n_ctx - first - 1, + /// scoring targets [first + 1, n_ctx) — the token at index first is context, never + /// a target. Scoring it too yields n_ctx/2 targets where llama.cpp has + /// n_ctx/2 - 1, which is a different measurement wearing the same name. /// /// Window size. /// Corpus token cap; 0 for unbounded. @@ -87,7 +92,7 @@ public readonly record struct PerplexityOptions( /// public static PerplexityOptions LlamaCppDefault(int contextLength, int maxTokens = 0, int bosTokenId = -1) => new(PerplexityMode.SlidingWindow, contextLength, Stride: contextLength, maxTokens, - UnscoredPrefix: contextLength / 2, BosTokenId: bosTokenId); + UnscoredPrefix: contextLength / 2 + 1, BosTokenId: bosTokenId); } /// Outcome of a perplexity run. @@ -103,8 +108,22 @@ public static PerplexityOptions LlamaCppDefault(int contextLength, int maxTokens /// matches: a perplexity computed over a different token count is a different measurement. /// /// Number of forward windows evaluated. +/// +/// One standard error of , from the sample variance of the per-token NLL: +/// sqrt(Var(nll) / (ScoredTokens - 1)) * Perplexity. 0 when fewer than two tokens +/// were scored, or when the variance is non-positive through rounding. +/// +/// +/// Read the error bar before comparing two perplexities. It is the difference between +/// a real regression and sampling noise, and on short corpora it is far wider than intuition +/// suggests: on a 2,286-token corpus this model's figure carries roughly ±6.5%, so a 3% +/// "discrepancy" against another implementation is not evidence of anything. The same model and +/// corpus family at wikitext-2 scale (150,195 scored tokens) narrows that to ±0.8%. +/// Matches llama.cpp's +/- figure, so the two are directly comparable. +/// public readonly record struct PerplexityResult( double Perplexity, double MeanNegativeLogLikelihood, int ScoredTokens, - int WindowCount); + int WindowCount, + double StandardError = 0); diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs index 6605905d..cb03d34d 100644 --- a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -49,6 +49,57 @@ public static PerplexityResult Evaluate( }; } + /// + /// Streaming mean and variance of the per-token NLL, by Welford's algorithm. + /// + /// + /// Deliberately not the textbook E[x²] - E[x]²: the per-token NLL of a converged model is + /// tightly clustered about a mean of a few nats, so the two terms agree to most of their + /// significant digits and their difference is mostly rounding error. On a constant NLL that form + /// returns ~1e-6 where the true variance is 0 — small in absolute terms, but it is the whole + /// signal. Welford accumulates the deviation directly and stays exact there, while agreeing with + /// the closed form (and so with llama.cpp's figure) everywhere else. + /// + private struct NllAccumulator + { + private double _mean; + private double _sumSquaredDeviation; + + public int Count { get; private set; } + + public double Mean => _mean; + + public void Add(double value) + { + Count++; + double delta = value - _mean; + _mean += delta / Count; + _sumSquaredDeviation += delta * (value - _mean); + } + + /// + /// One standard error of exp(Mean): the standard error of the mean NLL, carried + /// through exp by the delta method. Matches llama.cpp's +/- figure. + /// + public readonly double StandardErrorOfPerplexity(double perplexity) + { + if (Count < 2) + return 0; + double variance = _sumSquaredDeviation / Count; + return variance > 0 ? Math.Sqrt(variance / (Count - 1)) * perplexity : 0; + } + } + + /// Builds the result from the accumulated per-token NLL statistics. + /// Accumulated per-token NLL statistics. + /// Number of forward windows evaluated. + private static PerplexityResult Summarize(in NllAccumulator nll, int windows) + { + double perplexity = Math.Exp(nll.Mean); + return new PerplexityResult( + perplexity, nll.Mean, nll.Count, windows, nll.StandardErrorOfPerplexity(perplexity)); + } + private static PerplexityResult EvaluateTeacherForced( IPerplexityModel model, ReadOnlySpan tokens, int context) => model.ReturnsAllRows @@ -66,18 +117,15 @@ private static unsafe PerplexityResult TeacherForcedGrowingPrefix( var positions = new int[length]; for (int i = 0; i < length; i++) positions[i] = i; - double sumNll = 0; - int scored = 0; + var accumulator = new NllAccumulator(); for (int prefix = 1; prefix < length; prefix++) { using ITensor logits = model.Forward(tokens[..prefix], positions.AsSpan(0, prefix)); var row = new ReadOnlySpan((void*)logits.DataPointer, vocab); - sumNll += -LogProb.OfTarget(row, tokens[prefix]); - scored++; + accumulator.Add(-LogProb.OfTarget(row, tokens[prefix])); } - double meanNll = sumNll / scored; - return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); + return Summarize(accumulator, windows: accumulator.Count); } // Window w starts at w*Stride and covers [start, start + L); it scores the absolute targets @@ -114,8 +162,8 @@ private static unsafe PerplexityResult EvaluateSlidingWindow( int vocab = model.VocabSize; var positions = new int[context]; - double sumNll = 0; - int scored = 0, windows = 0; + var accumulator = new NllAccumulator(); + int windows = 0; // Positions restart at 0 for every window: each is an independent sequence, exactly as // llama.cpp evaluates each chunk. This is also what lets a corpus longer than the model's @@ -156,19 +204,17 @@ private static unsafe PerplexityResult EvaluateSlidingWindow( double nll = -LogProb.OfTarget(span, tokens[t]); windowNll += nll; windowScored++; - sumNll += nll; - scored++; + accumulator.Add(nll); } onWindow?.Invoke(windows - 1, Math.Exp(windowNll / windowScored), windowScored); } - if (scored == 0) + if (accumulator.Count == 0) throw new ArgumentException( $"Corpus of {tokens.Length} tokens is shorter than one context window of {context}.", nameof(tokens)); - double meanNll = sumNll / scored; - return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, windows); + return Summarize(accumulator, windows); } // Backend returns every row, so one forward pass scores every target: row i predicts token i+1. @@ -179,8 +225,7 @@ private static unsafe PerplexityResult TeacherForcedSinglePass( Span positions = length <= 512 ? stackalloc int[length] : new int[length]; for (int i = 0; i < length; i++) positions[i] = i; - double sumNll = 0; - int scored = 0; + var accumulator = new NllAccumulator(); using ITensor logits = model.Forward(tokens[..length], positions); int vocab = model.VocabSize; @@ -189,11 +234,9 @@ private static unsafe PerplexityResult TeacherForcedSinglePass( { var row = new ReadOnlySpan( (void*)(logits.DataPointer + (nint)i * vocab * sizeof(float)), vocab); - sumNll += -LogProb.OfTarget(row, tokens[i + 1]); - scored++; + accumulator.Add(-LogProb.OfTarget(row, tokens[i + 1])); } - double meanNll = sumNll / scored; - return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: 1); + return Summarize(accumulator, windows: 1); } } diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs index 6981abf4..ba0126d5 100644 --- a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -174,17 +174,88 @@ public void LlamaCppDefault_UsesNonOverlappingChunksScoringTheSecondHalf() var tokens = Enumerable.Range(0, 40).ToArray(); using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); - // L=16 => chunks at 0, 16 (32 would need tokens up to 48). Each scores its last 8. + // L=16 => chunks at 0, 16 (32 would need tokens up to 48). Each scores 7, not 8: + // llama.cpp's count is n_ctx - n_ctx/2 - 1. var result = PerplexityEvaluator.Evaluate( model, tokens, PerplexityOptions.LlamaCppDefault(contextLength: 16)); Assert.Equal(2, result.WindowCount); - Assert.Equal(16, result.ScoredTokens); // 2 chunks x 8 scored + Assert.Equal(14, result.ScoredTokens); // 2 chunks x 7 scored Assert.Equal(2, model.ForwardCalls.Count); Assert.Equal(0, model.ForwardCalls[0][0]); Assert.Equal(16, model.ForwardCalls[1][0]); // advances by the FULL window, not by 8 } + [Fact] + public void LlamaCppDefault_ScoresOneFewerTargetThanHalfTheWindow() + { + // llama.cpp: first = n_ctx/2, then `count += n_ctx - first - 1`, and the target for row j is + // token j+1 — so the token AT n_ctx/2 is context only and is never scored. Scoring it too + // would give n_ctx/2 targets per chunk instead of n_ctx/2 - 1: the same name, a different + // measurement, and a figure that is not comparable to a published one. + foreach (int context in new[] { 16, 64, 512 }) + { + var options = PerplexityOptions.LlamaCppDefault(context); + Assert.Equal(context / 2 + 1, options.UnscoredPrefix); + Assert.Equal(context / 2 - 1, context - options.UnscoredPrefix); + } + } + + [Fact] + public void StandardError_MatchesTheSampleVarianceOfPerTokenNll() + { + // Every target is token 0, and every row's only non-zero logit is at index 0 — so the + // target is always the argmax and its NLL depends solely on the row's peak height. That + // makes the whole set of per-token NLLs predictable in closed form, without reaching + // into the evaluator or replaying tensors. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[0] = position % 2 == 0 ? 2f : 0f; + return row; + } + + static double Nll(int position) + { + double peak = position % 2 == 0 ? 2.0 : 0.0; + return Math.Log(Math.Exp(peak) + (Vocab - 1)) - peak; + } + + var tokens = new int[40]; // all zero + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + + var result = PerplexityEvaluator.Evaluate(model, tokens, PerplexityOptions.LlamaCppDefault(16)); + + // L=16 => windows at 0 and 16; prefix 9 scores targets [s+9, s+16), i.e. rows 8..14. + var expectedNlls = new List(); + for (int window = 0; window < 2; window++) + for (int row = 8; row <= 14; row++) + expectedNlls.Add(Nll(row)); + + double mean = expectedNlls.Average(); + double variance = expectedNlls.Sum(v => (v - mean) * (v - mean)) / expectedNlls.Count; + double expected = Math.Sqrt(variance / (expectedNlls.Count - 1)) * Math.Exp(mean); + + Assert.Equal(expectedNlls.Count, result.ScoredTokens); + Assert.Equal(mean, result.MeanNegativeLogLikelihood, 9); + Assert.Equal(expected, result.StandardError, 9); + Assert.True(result.StandardError > 0); + } + + [Fact] + public void StandardError_IsZeroWhenEveryScoredTokenHasTheSameNll() + { + // A uniform distribution gives every target an identical NLL: zero variance, so the error + // bar must be exactly 0 rather than a NaN out of a negative rounding residue. + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate(model, tokens, PerplexityOptions.LlamaCppDefault(16)); + + Assert.Equal(0, result.StandardError); + } + + [Fact] public void LlamaCppDefault_AndContiguousTiling_ScoreDifferentTokenSets() { From c00068a7351a330ecd2c87180de4915f5efadfff Mon Sep 17 00:00:00 2001 From: James Burton Date: Fri, 31 Jul 2026 10:08:00 +0100 Subject: [PATCH 14/14] docs(eval): correct four contract/doc mismatches; cut chunks at whitespace-run starts (#231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From review feedback. Five of the six points were documentation that had drifted from the implementation — bad anywhere, worse here, since this harness's entire value is that its stated geometry is exactly what it measures. - IPerplexityModel.Forward described `positions` as absolute and invited implementers to rely on that. Sliding-window has passed window-relative positions restarting at 0 since e303be4a (llama.cpp chunk semantics, and what lets a corpus longer than the model's max sequence length be scored at all). Contract now says so. - PerplexityMode.TeacherForced claimed "one window per stride step"; it scores one clamped window and ignores Stride/UnscoredPrefix. Documented as deliberate — the mode reproduces the pre-existing harnesses bit for bit, and giving it geometry would move the numbers their gates were calibrated against. - The CLI's remarks said stride = context/2. The default is stride = context with unscored prefix = context/2 + 1. Also corrected --unscored-prefix's own help text, which still said context/2 after ca8a92bf changed it. The sixth was behavioural and is fixed: chunks were cut at the last whitespace *character*, which can land inside a run of them. A GPT-2-style pre-tokenizer treats a whitespace run as one unit, so splitting one yields a different stream than tokenizing the file in one pass. Cuts now land at the run's START, so the whole run moves into the carry. Verified against whole-file tokenization on wikitext-2 (1.29 MB, 301,948 tokens) at chunk sizes 997 / 4096 / 8192 / 65536 — identical streams every time. The smallest exercises ~1,290 boundaries; the previously-reported validation crossed only ~20, which was thin cover for a boundary bug. Tests: whitespace runs of spaces, mixed space/tab/newline, newlines-only and CRLF all assert no segment ends in whitespace. The whitespace-free case is asserted too, since it has no safe cut and is therefore read whole — a stated trade (correctness over bounded memory) rather than an accident. --- src/DotLLM.Cli/Commands/PerplexityCommand.cs | 12 ++-- .../Evaluation/IPerplexityModel.cs | 9 ++- .../Evaluation/PerplexityResult.cs | 17 ++++-- src/DotLLM.Engine/Evaluation/CorpusReader.cs | 41 ++++++++++++-- .../Evaluation/CorpusReaderTests.cs | 56 +++++++++++++++++++ 5 files changed, 118 insertions(+), 17 deletions(-) diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs index b82f703f..e5c39783 100644 --- a/src/DotLLM.Cli/Commands/PerplexityCommand.cs +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -17,9 +17,13 @@ namespace DotLLM.Cli.Commands; /// Computes perplexity over a text corpus: load → stream-tokenize → score. /// /// -/// Defaults to with stride = context / 2, which -/// reproduces llama.cpp's --perplexity methodology, so the reported figure is directly -/// comparable to published numbers for the same model, corpus, context and stride. +/// Defaults to with stride = context and +/// unscored prefix = context / 2 + 1 — non-overlapping chunks, each scoring the targets after +/// its midpoint. That reproduces llama.cpp's --perplexity methodology, so the reported figure +/// is directly comparable to published numbers for the same model, corpus and context. +/// Advance and scored span are separate knobs on purpose: llama.cpp advances by the whole +/// window yet scores only part of it, so its scored ranges have gaps. A single "stride" cannot +/// express that. /// internal sealed class PerplexityCommand : AsyncCommand { @@ -50,7 +54,7 @@ public sealed class Settings : CommandSettings public int Stride { get; set; } [CommandOption("--unscored-prefix")] - [Description("Leading tokens of each window used as context only. -1 selects context/2 (llama.cpp's default).")] + [Description("Leading tokens of each window used as context only. -1 selects context/2 + 1, which scores the same targets as llama.cpp.")] [DefaultValue(-1)] public int UnscoredPrefix { get; set; } = -1; diff --git a/src/DotLLM.Core/Evaluation/IPerplexityModel.cs b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs index fd7f0a00..71e57141 100644 --- a/src/DotLLM.Core/Evaluation/IPerplexityModel.cs +++ b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs @@ -46,8 +46,13 @@ public interface IPerplexityModel /// /// Input token ids for this window. /// - /// Absolute position ids, one per token. Passed explicitly rather than derived, so a sliding - /// window can score at its true absolute positions instead of restarting at zero. + /// Position ids, one per token, passed explicitly rather than derived so the caller controls + /// them. + /// Do not assume they are absolute corpus offsets. Sliding-window scoring passes + /// window-relative positions restarting at 0 for every window, because llama.cpp + /// evaluates each chunk as an independent sequence and matching it is the point of that mode. + /// It is also what lets a corpus longer than be scored at all — + /// absolute positions would run past the model's limit on the second window. /// /// /// Logits, owned by the caller. Row layout is governed by . diff --git a/src/DotLLM.Core/Evaluation/PerplexityResult.cs b/src/DotLLM.Core/Evaluation/PerplexityResult.cs index 52782c5b..bec785e1 100644 --- a/src/DotLLM.Core/Evaluation/PerplexityResult.cs +++ b/src/DotLLM.Core/Evaluation/PerplexityResult.cs @@ -7,14 +7,19 @@ namespace DotLLM.Core.Evaluation; public enum PerplexityMode { /// - /// Teacher-forced scoring over one window per stride step, each scored from a single forward - /// pass where the backend permits it. The established in-tree methodology (the "G1 precedent" - /// referenced by the CUDA prefill harnesses); preserved so existing quality gates keep their - /// meaning after consolidation. + /// Teacher-forced scoring over a single window — the first + /// tokens, clamped to the model's maximum — scored from one forward pass where the backend + /// permits it. The established in-tree methodology (the "G1 precedent" referenced by the CUDA + /// prefill harnesses); preserved so existing quality gates keep their meaning after consolidation. /// /// - /// Ratio-oriented: the load-bearing signal is the OFF/ON perplexity ratio on identical tokens, - /// not the absolute value. Not comparable to published figures. + /// and + /// are ignored in this mode: it scores one window and stops. That is deliberate — the mode + /// exists to reproduce the pre-existing harnesses bit for bit, and giving it window geometry + /// would change the numbers those gates were calibrated against. Use + /// to walk a corpus. + /// Ratio-oriented: the load-bearing signal is the OFF/ON perplexity ratio on identical + /// tokens, not the absolute value. Not comparable to published figures. /// TeacherForced, diff --git a/src/DotLLM.Engine/Evaluation/CorpusReader.cs b/src/DotLLM.Engine/Evaluation/CorpusReader.cs index 12ea325e..5418ee27 100644 --- a/src/DotLLM.Engine/Evaluation/CorpusReader.cs +++ b/src/DotLLM.Engine/Evaluation/CorpusReader.cs @@ -17,8 +17,20 @@ public static class CorpusReader /// ids in order, stopping after (0 = unbounded). /// /// - /// Chunks are cut at the last whitespace so a token is never split across a boundary; the - /// remainder is carried into the next chunk, and the final carry is flushed whole. + /// Chunks are cut immediately before the last run of whitespace, so a token is never split + /// across a boundary; the remainder is carried into the next chunk, and the final carry is + /// flushed whole. Verified against whole-file tokenization on wikitext-2 (1.29 MB, 301,948 + /// tokens) at chunk sizes from 997 to 65536 — identical streams, so ~1,290 boundaries in the + /// smallest case land in awkward places without changing a single id. + /// Whole runs, not single characters. Cutting at the last whitespace character + /// can fall inside a run of them, and a GPT-2-style pre-tokenizer treats a whitespace run as one + /// unit — splitting it yields a different token stream than tokenizing the file in one pass, + /// which is precisely the silent divergence this harness exists to rule out. + /// Known limitation: text containing no whitespace at all accumulates in the carry + /// buffer until some arrives, or until the corpus ends. Flushing at an arbitrary character + /// instead would bound the memory but change the token stream, so correctness wins here. Ordinary + /// prose corpora are unaffected; a whitespace-free corpus (minified JSON, unsegmented CJK) is + /// effectively read whole. /// /// Corpus source. /// Tokenizer whose vocabulary the ids belong to. @@ -43,10 +55,10 @@ public static IEnumerable StreamTokens( carry.Append(buffer, 0, read); string pending = carry.ToString(); - int cut = pending.LastIndexOf(' '); - if (cut < 0) continue; // no safe split point yet; keep accumulating + int cut = LastWhitespaceRunStart(pending); + if (cut <= 0) continue; // no safe split point yet; keep accumulating - // The separating space is carried INTO the next chunk, not dropped. GPT-2-style BPE + // The separating whitespace is carried INTO the next chunk, not dropped. GPT-2-style BPE // encodes a leading space as part of the following token, so dropping it silently // changes the token stream — and therefore the perplexity — versus tokenizing the // corpus in one pass. @@ -70,4 +82,23 @@ public static IEnumerable StreamTokens( } } } + + /// + /// Index of the first character of the last whitespace run in , or + /// -1 when it contains no whitespace. + /// + /// + /// Returning the run's START rather than the last whitespace character is what keeps the run + /// intact: everything from it onwards moves into the carry, so no pre-token that spans a + /// whitespace run is ever cut in half. + /// + private static int LastWhitespaceRunStart(string text) + { + int i = text.Length - 1; + while (i >= 0 && !char.IsWhiteSpace(text[i])) i--; + if (i < 0) return -1; + + while (i > 0 && char.IsWhiteSpace(text[i - 1])) i--; + return i; + } } diff --git a/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs index fb553e1e..5428f415 100644 --- a/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs +++ b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs @@ -44,4 +44,60 @@ public void StreamTokens_DoesNotSplitTokensAcrossChunkBoundaries() var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 0, charChunkSize: 4).ToArray(); Assert.Equal([1, 2, 3, 4, 5, 6], tokens); } + + /// Records the exact strings handed to the tokenizer, so cut points are assertable. + private sealed class RecordingTokenizer : ITokenizer + { + public List Segments { get; } = []; + + public int[] Encode(string text) + { + Segments.Add(text); + return [text.Length]; + } + + public string Decode(ReadOnlySpan tokenIds) => throw new NotSupportedException(); + public string DecodeToken(int tokenId) => throw new NotSupportedException(); + public int CountTokens(string text) => 1; + public int VocabSize => 1024; + public int BosTokenId => 0; + public int EosTokenId => 1; + } + + [Theory] + [InlineData("aa bb")] // run of spaces + [InlineData("aa \n\t bb")] // mixed whitespace run + [InlineData("aa\n\n\nbb")] // run of newlines, no spaces at all + [InlineData("aa\r\nbb")] // CRLF + public void StreamTokens_NeverCutsInsideAWhitespaceRun(string text) + { + // A GPT-2-style pre-tokenizer treats a whitespace run as one unit. Cutting inside one makes + // the streamed token stream differ from tokenizing the file in a single pass — a silent + // divergence that would invalidate every comparison this harness exists to support. + // Cutting on the last whitespace *character* rather than the run's start does exactly that, + // and a corpus of newlines with no spaces is not cut at all. + var tokenizer = new RecordingTokenizer(); + using var reader = new StringReader(text); + _ = CorpusReader.StreamTokens(reader, tokenizer, maxTokens: 0, charChunkSize: 4).ToArray(); + + Assert.Equal(text, string.Concat(tokenizer.Segments)); + Assert.All(tokenizer.Segments, s => Assert.False( + s.Length > 0 && char.IsWhiteSpace(s[^1]), + $"segment '{s.Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t")}' " + + "ends in whitespace, so a whitespace run was cut in half")); + } + + [Fact] + public void StreamTokens_WhitespaceFreeInputIsEmittedWhole() + { + // Documented limitation: with no whitespace there is no safe cut, so the carry accumulates + // and the corpus is effectively read whole. Asserted so the behaviour is a stated trade + // rather than a surprise. + var tokenizer = new RecordingTokenizer(); + using var reader = new StringReader("abcdefghijklmnop"); + _ = CorpusReader.StreamTokens(reader, tokenizer, maxTokens: 0, charChunkSize: 4).ToArray(); + + Assert.Single(tokenizer.Segments); + Assert.Equal("abcdefghijklmnop", tokenizer.Segments[0]); + } }