diff --git a/docs/superpowers/plans/2026-07-30-perplexity-harness.md b/docs/superpowers/plans/2026-07-30-perplexity-harness.md new file mode 100644 index 00000000..d53ed34d --- /dev/null +++ b/docs/superpowers/plans/2026-07-30-perplexity-harness.md @@ -0,0 +1,956 @@ +# Perplexity Harness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a shared perplexity harness with a llama.cpp-comparable sliding-window mode, so numerics-changing kernel work (ours and upstream's) has an evaluation gate. + +**Architecture:** Abstractions in `DotLLM.Core.Evaluation` (already landed, commit `7af01b73`); a single `PerplexityEvaluator` that selects its execution strategy from `IPerplexityModel.ReturnsAllRows` rather than from the caller; a streaming corpus reader; a `TransformerModel` adapter; and a `dotllm perplexity` CLI verb. + +**Tech Stack:** .NET 10, C#, xUnit (+ `Xunit.SkippableFact`), Spectre.Console `CommandApp` for CLI. + +## Global Constraints + +- Branch `issue/231-perplexity-harness`, worktree `.claude/worktrees/ppl-harness`, based on `main`. PR targets upstream `main`. +- File-scoped namespaces. `enable`. XML doc comments on all public APIs. +- Commit messages include `(#231)` and end with `Co-Authored-By: Claude Opus 5 (1M context) `. +- **The evaluator never loads weights.** It accepts an already-constructed `IPerplexityModel`. Non-negotiable: it is the Track D constraint. +- **The corpus is streamed and tokenized in chunks**, never materialized whole. +- `TeacherForced` mode must be preserved verbatim in behaviour. It is the "G1 precedent" methodology that existing quality gates depend on; changing any number it produces defeats the migration gate. +- No `System.Linq` on scoring hot paths; no managed allocation per scored token. + +--- + +## Methodology note: what "llama.cpp-comparable" means precisely + +llama.cpp's `perplexity` walks the corpus in chunks of `n_ctx` and scores **only the second half** of each chunk, so every scored token has at least `n_ctx/2` tokens of preceding context. The first half of the very first chunk is never scored. + +Generalised to `(ContextLength L, Stride S)`: + +- Window `w` covers absolute token range `[w*S, w*S + L)`. +- Scored targets are absolute indices `t` in `[w*S + L - S, w*S + L)`. +- Each scored token therefore has `L - S` tokens of context, and consecutive windows tile the scored tokens exactly — no gaps, no double-counting. +- **`S = L/2` reproduces llama.cpp's default exactly.** + +A forward pass over window `[s, s+L)` returns rows `0..L-1`, where row `i` holds the distribution predicting the token at absolute index `s+i+1`. So scoring absolute target `t` reads row `t-s-1`. This requires `L - S >= 1`. + +Tokens before the first window's scored range are never scored, matching llama.cpp. This is a known small bias and is documented rather than silently "fixed", because fixing it would break comparability. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `src/DotLLM.Core/Evaluation/IPerplexityModel.cs` | **Landed.** Model contract. | +| `src/DotLLM.Core/Evaluation/PerplexityResult.cs` | **Landed.** `PerplexityMode`, `PerplexityOptions`, `PerplexityResult`. | +| `src/DotLLM.Engine/Evaluation/LogProb.cs` | Numerically stable log-softmax of a single logit row. | +| `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` | Both modes; strategy chosen from `ReturnsAllRows`. | +| `src/DotLLM.Engine/Evaluation/CorpusReader.cs` | Streaming corpus → token chunks. | +| `src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs` | `IPerplexityModel` over `TransformerModel`. | +| `src/DotLLM.Cli/Commands/PerplexityCommand.cs` | `dotllm perplexity` verb. | +| `tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs` | Deterministic test double. | +| `tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs` | Log-softmax correctness. | +| `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` | Both modes, both backend shapes. | +| `tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs` | Streaming/chunking. | +| `tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs` | llama.cpp figure validation. | + +--- + +### Task 1: Stable log-probability + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/LogProb.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs` + +**Interfaces:** +- Consumes: nothing. +- Produces: `static double LogProb.OfTarget(ReadOnlySpan logits, int target)` — log-softmax value at `target`, in nats. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class LogProbTests +{ + [Fact] + public void OfTarget_UniformLogits_IsNegativeLogVocab() + { + // Four equal logits => p = 1/4 for each => log p = -log 4. + var logits = new float[] { 2.5f, 2.5f, 2.5f, 2.5f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.Equal(-Math.Log(4.0), actual, 12); + } + + [Fact] + public void OfTarget_IsShiftInvariant() + { + var a = new float[] { 1f, 2f, 3f }; + var b = new float[] { 1001f, 1002f, 1003f }; + Assert.Equal(LogProb.OfTarget(a, 1), LogProb.OfTarget(b, 1), 12); + } + + [Fact] + public void OfTarget_LargeLogits_DoesNotOverflow() + { + // Naive exp() would overflow to infinity here; the max-shift must prevent it. + var logits = new float[] { 800f, 900f, 1000f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.True(double.IsFinite(actual)); + Assert.Equal(0.0, actual, 6); // target dominates => p ~ 1 => log p ~ 0 + } + + [Fact] + public void OfTarget_SumOfProbabilitiesIsOne() + { + var logits = new float[] { -1.5f, 0.25f, 3f, 0.5f }; + double sum = 0; + for (int i = 0; i < logits.Length; i++) sum += Math.Exp(LogProb.OfTarget(logits, i)); + Assert.Equal(1.0, sum, 10); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~LogProbTests"` +Expected: FAIL — `LogProb` does not exist (compile error). + +- [ ] **Step 3: Write minimal implementation** + +```csharp +namespace DotLLM.Engine.Evaluation; + +/// Numerically stable log-softmax over a single row of logits. +public static class LogProb +{ + /// + /// Returns log P(target) in nats under a softmax over . + /// + /// + /// Uses the max-shift identity log softmax(x)_t = (x_t - m) - log sum_j exp(x_j - m) + /// with m = max(x), so no exp argument is ever positive and overflow is + /// impossible. Accumulates in : a vocab of 128k float32 terms loses + /// meaningful precision in float32, and perplexity differences between near-identical runs + /// are exactly what this harness exists to resolve. + /// + public static double OfTarget(ReadOnlySpan logits, int target) + { + if ((uint)target >= (uint)logits.Length) + throw new ArgumentOutOfRangeException(nameof(target)); + + float max = logits[0]; + for (int j = 1; j < logits.Length; j++) + if (logits[j] > max) max = logits[j]; + + double sumExp = 0; + for (int j = 0; j < logits.Length; j++) + sumExp += Math.Exp(logits[j] - max); + + return (logits[target] - max) - Math.Log(sumExp); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~LogProbTests"` +Expected: PASS, 4 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/LogProb.cs tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs +git commit -m "feat(eval): numerically stable log-softmax for perplexity scoring (#231)" +``` + +--- + +### Task 2: Test double + +**Files:** +- Create: `tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs` + +**Interfaces:** +- Consumes: `IPerplexityModel` from `DotLLM.Core.Evaluation`. +- Produces: `FakePerplexityModel(int vocabSize, int maxContextLength, bool returnsAllRows, Func rowFactory)` with `IReadOnlyList ForwardCalls` recording every window passed to `Forward`. + +The recording matters as much as the logits: Tasks 3–5 assert on *which windows were evaluated*, which is how window tiling and the O(n) vs O(n²) strategy split get tested without a real model. + +- [ ] **Step 1: Write the implementation** (a test double, so no test-first cycle) + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Tests.Unit.Evaluation; + +/// +/// Deterministic for evaluator tests. Records every window it is +/// asked to score so tests can assert on window tiling, not just on the resulting number. +/// +internal sealed class FakePerplexityModel : IPerplexityModel, IDisposable +{ + private readonly Func _rowFactory; // (absolutePosition, vocabSize) => logits + private readonly List _forwardCalls = new(); + private readonly List _issued = new(); + + public FakePerplexityModel( + int vocabSize, int maxContextLength, bool returnsAllRows, + Func rowFactory) + { + VocabSize = vocabSize; + MaxContextLength = maxContextLength; + ReturnsAllRows = returnsAllRows; + _rowFactory = rowFactory; + } + + public int VocabSize { get; } + public int MaxContextLength { get; } + public bool ReturnsAllRows { get; } + + /// Token windows passed to , in call order. + public IReadOnlyList ForwardCalls => _forwardCalls; + + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + { + _forwardCalls.Add(tokens.ToArray()); + + int rows = ReturnsAllRows ? tokens.Length : 1; + int firstRow = ReturnsAllRows ? 0 : tokens.Length - 1; + var tensor = UnmanagedTensor.Allocate(new TensorShape(rows, VocabSize), DType.F32); + unsafe + { + var dest = new Span((void*)tensor.DataPointer, rows * VocabSize); + for (int r = 0; r < rows; r++) + _rowFactory(positions[firstRow + r], VocabSize).CopyTo(dest[(r * VocabSize)..]); + } + _issued.Add(tensor); + return tensor; + } + + /// Uniform logits: every target scores exactly -log(vocabSize). + public static Func Uniform => (_, vocab) => new float[vocab]; + + public void Dispose() + { + foreach (var t in _issued) t.Dispose(); + _issued.Clear(); + } +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `dotnet build tests/DotLLM.Tests.Unit` +Expected: build succeeds. If `UnmanagedTensor.Allocate` differs in signature, adjust to the actual factory in `src/DotLLM.Core/Tensors/UnmanagedTensor.cs` — do not change the tensor type used. + +- [ ] **Step 3: Commit** + +```bash +git add tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs +git commit -m "test(eval): deterministic IPerplexityModel double recording forward windows (#231)" +``` + +--- + +### Task 3: `TeacherForced` mode, all-rows backend + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: `LogProb.OfTarget`, `IPerplexityModel`, `PerplexityOptions`, `PerplexityResult`, `FakePerplexityModel`. +- Produces: `PerplexityResult PerplexityEvaluator.Evaluate(IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options)`. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class PerplexityEvaluatorTests +{ + private static readonly int[] Tokens = Enumerable.Range(0, 32).ToArray(); + + [Fact] + public void TeacherForced_AllRowsBackend_UniformLogitsGivesVocabSizePerplexity() + { + // Uniform logits => P(any target) = 1/vocab => perplexity == vocab, exactly. + using var model = new FakePerplexityModel( + vocabSize: 7, maxContextLength: 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, ContextLength: 32, Stride: 32)); + + Assert.Equal(7.0, result.Perplexity, 9); + Assert.Equal(31, result.ScoredTokens); // n-1 targets from one pass + } + + [Fact] + public void TeacherForced_AllRowsBackend_UsesASingleForwardPass() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // The whole point of ReturnsAllRows: one pass scores every target. + Assert.Single(model.ForwardCalls); + Assert.Equal(32, model.ForwardCalls[0].Length); + } + + [Fact] + public void MeanNll_AndPerplexity_AreConsistent() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + Assert.Equal(result.Perplexity, Math.Exp(result.MeanNegativeLogLikelihood), 9); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — `PerplexityEvaluator` does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Engine.Evaluation; + +/// +/// Computes perplexity over a token sequence using an . +/// +/// +/// The evaluator never loads weights — callers pass an already-constructed model. On unified-memory +/// parts a large VRAM carve-out leaves host RAM scarce, and perplexity (a long run of full-context +/// prefills) is the workload most punished by holding a second host-side copy of the weights. +/// +public static class PerplexityEvaluator +{ + /// Scores and returns the aggregate result. + public static PerplexityResult Evaluate( + IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options) + { + ArgumentNullException.ThrowIfNull(model); + if (tokens.Length < 2) + throw new ArgumentException("At least two tokens are required to score one target.", nameof(tokens)); + + int context = Math.Min(options.ContextLength, model.MaxContextLength); + if (context < 2) + throw new ArgumentException("Context length must be at least 2.", nameof(options)); + + return options.Mode switch + { + PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), + _ => throw new NotSupportedException($"Mode {options.Mode} is not implemented yet."), + }; + } + + private static unsafe PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + Span positions = length <= 512 ? stackalloc int[length] : new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + + using ITensor logits = model.Forward(tokens[..length], positions); + int vocab = model.VocabSize; + // Row i predicts token i+1, so the final row has no target within the window. + for (int i = 0; i < length - 1; i++) + { + var row = new ReadOnlySpan((void*)(logits.DataPointer + (nint)i * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(row, tokens[i + 1]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: 1); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): PerplexityEvaluator teacher-forced mode for all-rows backends (#231)" +``` + +--- + +### Task 4: `TeacherForced` mode, last-row-only backend + +**Files:** +- Modify: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: everything from Task 3. +- Produces: no new public surface. `Evaluate` now honours `ReturnsAllRows == false` via growing-prefix re-prefill. + +This is the O(n²) path the CUDA harnesses need. It must produce the **same number** as the all-rows path on the same tokens — that equivalence is what makes the migration gate meaningful. + +- [ ] **Step 1: Write the failing test** + +```csharp + [Fact] + public void TeacherForced_LastRowOnlyBackend_MatchesAllRowsBackendExactly() + { + // Position-dependent but deterministic logits, so a wrong row/position mapping shows up. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + for (int j = 0; j < vocab; j++) row[j] = (float)Math.Sin((position + 1) * (j + 1) * 0.37); + return row; + } + + using var allRows = new FakePerplexityModel(7, 64, returnsAllRows: true, Rows); + using var lastRow = new FakePerplexityModel(7, 64, returnsAllRows: false, Rows); + var options = new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32); + + var a = PerplexityEvaluator.Evaluate(allRows, Tokens, options); + var b = PerplexityEvaluator.Evaluate(lastRow, Tokens, options); + + Assert.Equal(a.Perplexity, b.Perplexity, 9); + Assert.Equal(a.ScoredTokens, b.ScoredTokens); + } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_ReprefixesGrowingWindows() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: false, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // One forward per scored target, each one token longer than the last. + Assert.Equal(31, model.ForwardCalls.Count); + for (int i = 0; i < model.ForwardCalls.Count; i++) + Assert.Equal(i + 1, model.ForwardCalls[i].Length); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — last-row-only path reads row 0 as if it were the full grid, so perplexity mismatches (or an index error). + +- [ ] **Step 3: Write minimal implementation** + +Replace `EvaluateTeacherForced` with a dispatcher plus the two strategies: + +```csharp + private static PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + => model.ReturnsAllRows + ? TeacherForcedSinglePass(model, tokens, context) + : TeacherForcedGrowingPrefix(model, tokens, context); + + // Backend returns only the final row, so each target needs its own prefill over the growing + // prefix. O(n^2) in forward passes — unavoidable, and the reason the CUDA harnesses that + // originated this methodology carry a stride. + private static unsafe PerplexityResult TeacherForcedGrowingPrefix( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + int vocab = model.VocabSize; + var positions = new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + double sumNll = 0; + int scored = 0; + for (int prefix = 1; prefix < length; prefix++) + { + using ITensor logits = model.Forward(tokens[..prefix], positions.AsSpan(0, prefix)); + var row = new ReadOnlySpan((void*)logits.DataPointer, vocab); + sumNll += -LogProb.OfTarget(row, tokens[prefix]); + scored++; + } + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, WindowCount: scored); + } +``` + +Rename the Task 3 body to `TeacherForcedSinglePass` (signature unchanged). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 5 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): growing-prefix teacher-forced path for last-row-only backends (#231)" +``` + +--- + +### Task 5: `SlidingWindow` mode + +**Files:** +- Modify: `src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs` + +**Interfaces:** +- Consumes: everything above. +- Produces: no new public surface. `PerplexityMode.SlidingWindow` becomes functional. + +Implement exactly the tiling defined in the methodology note at the top of this plan. Re-read it before writing code. + +- [ ] **Step 1: Write the failing test** + +```csharp + [Fact] + public void SlidingWindow_TilesScoredTokensWithoutGapsOrOverlap() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16, S=8 => windows start at 0, 8, 16, 24; each scores its last 8 targets. + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 8)); + + Assert.Equal(4, result.WindowCount); + Assert.Equal(32, result.ScoredTokens); // 4 windows x 8 targets + Assert.Equal(7.0, result.Perplexity, 9); // uniform logits + + Assert.Equal(4, model.ForwardCalls.Count); + Assert.All(model.ForwardCalls, w => Assert.Equal(16, w.Length)); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(8, model.ForwardCalls[1][0]); + Assert.Equal(16, model.ForwardCalls[2][0]); + Assert.Equal(24, model.ForwardCalls[3][0]); + } + + [Fact] + public void SlidingWindow_ScoresEachTargetAtItsTrueAbsolutePosition() + { + // Logits keyed to absolute position: a window that restarts positions at zero scores + // different values and fails this. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[position % vocab] = 10f; + return row; + } + + var tokens = new int[40]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = (i + 1) % 7; // target == predicted argmax + + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, Rows); + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, 8)); + + // Row i of window [s, s+L) sits at absolute position s+i and predicts token s+i+1, + // whose id is (s+i+1)%7 -- exactly the argmax. So NLL is near zero throughout. + Assert.True(result.MeanNegativeLogLikelihood < 0.01, + $"expected confident predictions, got mean NLL {result.MeanNegativeLogLikelihood}"); + } + + [Fact] + public void SlidingWindow_RejectsStrideNotSmallerThanContext() + { + using var model = new FakePerplexityModel(7, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + Assert.Throws(() => PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16))); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: FAIL — `NotSupportedException` from the `Evaluate` switch. + +- [ ] **Step 3: Write minimal implementation** + +Add the `SlidingWindow` arm to the `Evaluate` switch, then: + +```csharp + // llama.cpp `--perplexity` tiling. Window w covers [w*S, w*S + L) and scores absolute targets + // in [w*S + L - S, w*S + L), so every scored token carries L-S tokens of context and the + // scored ranges tile the corpus exactly. S = L/2 reproduces llama.cpp's default. + // Targets before the first window's scored range are never scored -- llama.cpp skips them too, + // and "fixing" that would break comparability. + private static unsafe PerplexityResult EvaluateSlidingWindow( + IPerplexityModel model, ReadOnlySpan tokens, int context, int stride) + { + if (stride < 1 || stride >= context) + throw new ArgumentException( + $"Stride must be in [1, {context - 1}] for a context of {context}; each scored token needs at least one token of context.", + nameof(stride)); + + int vocab = model.VocabSize; + var positions = new int[context]; + double sumNll = 0; + int scored = 0, windows = 0; + + for (int start = 0; start + context <= tokens.Length; start += stride) + { + for (int i = 0; i < context; i++) positions[i] = start + i; + + using ITensor logits = model.Forward(tokens.Slice(start, context), positions); + windows++; + + // Absolute targets [start + context - stride, start + context); row for target t is t-start-1. + for (int t = start + context - stride; t < start + context; t++) + { + int row = t - start - 1; + var span = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)row * vocab * sizeof(float)), vocab); + sumNll += -LogProb.OfTarget(span, tokens[t]); + scored++; + } + } + + if (scored == 0) + throw new ArgumentException( + $"Corpus of {tokens.Length} tokens is shorter than one context window of {context}.", nameof(tokens)); + + double meanNll = sumNll / scored; + return new PerplexityResult(Math.Exp(meanNll), meanNll, scored, windows); + } +``` + +Note: the last-row-only backend is **not** supported in `SlidingWindow` — throw `NotSupportedException` with a message pointing at `TeacherForced`, since re-prefilling per target inside a sliding window is both O(n²) and redundant with the growing-prefix mode. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~PerplexityEvaluatorTests"` +Expected: PASS, 8 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs +git commit -m "feat(eval): llama.cpp-comparable sliding-window perplexity mode (#231)" +``` + +--- + +### Task 6: Streaming corpus reader + +**Files:** +- Create: `src/DotLLM.Engine/Evaluation/CorpusReader.cs` +- Test: `tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs` + +**Interfaces:** +- Consumes: `DotLLM.Tokenizers` `ITokenizer` (`int[] Encode(string)`). +- Produces: `static IEnumerable CorpusReader.StreamTokens(TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536)`. + +Streaming is a spec constraint, not an optimisation: `wiki.test.raw` is ~1.3 MB and its token array is ~340k ints, and Track D's premise is that host RAM is the scarce resource. + +- [ ] **Step 1: Write the failing test** + +```csharp +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class CorpusReaderTests +{ + // One token per whitespace-separated word; ids are word lengths, so order is checkable. + private sealed class WordTokenizer : ITokenizer + { + public int[] Encode(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(w => w.Length).ToArray(); + public string Decode(ReadOnlySpan ids) => throw new NotSupportedException(); + } + + [Fact] + public void StreamTokens_ProducesTokensInOrder() + { + using var reader = new StringReader("a bb ccc dddd"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer()).ToArray(); + Assert.Equal(new[] { 1, 2, 3, 4 }, tokens); + } + + [Fact] + public void StreamTokens_HonoursMaxTokens() + { + using var reader = new StringReader("a bb ccc dddd eeeee"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 3).ToArray(); + Assert.Equal(new[] { 1, 2, 3 }, tokens); + } + + [Fact] + public void StreamTokens_DoesNotSplitTokensAcrossChunkBoundaries() + { + // A tiny chunk size forces the boundary case: "ccc" must not become "c" + "cc". + using var reader = new StringReader("a bb ccc dddd eeeee ffffff"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 0, charChunkSize: 4).ToArray(); + Assert.Equal(new[] { 1, 2, 3, 4, 5, 6 }, tokens); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~CorpusReaderTests"` +Expected: FAIL — `CorpusReader` does not exist. If `ITokenizer`'s real shape differs, adjust the stub to match `src/DotLLM.Tokenizers/` and keep the assertions. + +- [ ] **Step 3: Write minimal implementation** + +```csharp +using System.Text; +using DotLLM.Tokenizers; + +namespace DotLLM.Engine.Evaluation; + +/// Streams a text corpus into tokens without materializing the whole file or token array. +public static class CorpusReader +{ + /// + /// Reads in character chunks, tokenizes each chunk, and yields token + /// ids in order, stopping after (0 = unbounded). + /// + /// + /// Chunks are cut at the last whitespace so a token is never split across a boundary; the + /// remainder is carried into the next chunk. The final chunk is flushed whole. + /// + public static IEnumerable StreamTokens( + TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(tokenizer); + if (charChunkSize < 1) throw new ArgumentOutOfRangeException(nameof(charChunkSize)); + + var buffer = new char[charChunkSize]; + var carry = new StringBuilder(); + int emitted = 0; + + while (true) + { + int read = reader.Read(buffer, 0, buffer.Length); + if (read == 0) break; + + carry.Append(buffer, 0, read); + string pending = carry.ToString(); + + int cut = pending.LastIndexOf(' '); + if (cut < 0) continue; // no safe split point yet; accumulate + + string ready = pending[..cut]; + carry.Clear(); + carry.Append(pending[(cut + 1)..]); + + foreach (int id in tokenizer.Encode(ready)) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + + if (carry.Length > 0) + { + foreach (int id in tokenizer.Encode(carry.ToString())) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/DotLLM.Tests.Unit --filter "FullyQualifiedName~CorpusReaderTests"` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add src/DotLLM.Engine/Evaluation/CorpusReader.cs tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs +git commit -m "feat(eval): streaming corpus tokenizer for perplexity runs (#231)" +``` + +--- + +### Task 7: `TransformerModel` adapter + +**Files:** +- Create: `src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs` + +**Interfaces:** +- Consumes: `TransformerModel` (`Config.VocabSize`, `Config.MaxSequenceLength`, `ITensor Forward(ReadOnlySpan, ReadOnlySpan, int deviceId)`), `IPerplexityModel`. +- Produces: `TransformerPerplexityModel(TransformerModel model, int deviceId = -1)` implementing `IPerplexityModel`. + +- [ ] **Step 1: Write the implementation** + +`TransformerModel.Forward` is documented as returning `[seqLen, vocab]` for all input positions, so `ReturnsAllRows` is `true`. + +```csharp +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; +using DotLLM.Models.Architectures; + +namespace DotLLM.Models.Evaluation; + +/// Adapts to . +/// +/// Holds a borrowed reference: the adapter does not own the model and does not dispose it, so the +/// caller keeps a single resident copy of the weights. This is the whole point of the evaluator +/// taking a constructed model rather than a path. +/// +public sealed class TransformerPerplexityModel : IPerplexityModel +{ + private readonly TransformerModel _model; + private readonly int _deviceId; + + /// An already-loaded model. Not owned; not disposed by this adapter. + /// Device for the forward pass; -1 is CPU. + public TransformerPerplexityModel(TransformerModel model, int deviceId = -1) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _deviceId = deviceId; + } + + /// + public int VocabSize => _model.Config.VocabSize; + + /// + public int MaxContextLength => _model.Config.MaxSequenceLength; + + /// + public bool ReturnsAllRows => true; + + /// + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + => _model.Forward(tokens, positions, _deviceId); +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `dotnet build src/DotLLM.Models` +Expected: build succeeds. + +- [ ] **Step 3: Commit** + +```bash +git add src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs +git commit -m "feat(eval): IPerplexityModel adapter over TransformerModel (#231)" +``` + +--- + +### Task 8: `dotllm perplexity` CLI verb + +**Files:** +- Create: `src/DotLLM.Cli/Commands/PerplexityCommand.cs` +- Modify: `src/DotLLM.Cli/Program.cs` (register alongside `run`/`chat`/`serve`) + +**Interfaces:** +- Consumes: `PerplexityEvaluator.Evaluate`, `CorpusReader.StreamTokens`, `TransformerPerplexityModel`, `GgufFileResolver` (follow `RunCommand`'s resolution pattern exactly). +- Produces: CLI verb `dotllm perplexity --corpus [--context N] [--stride N] [--max-tokens N] [--mode sliding-window|teacher-forced]`. + +- [ ] **Step 1: Write the command** + +Follow `RunCommand`'s `AsyncCommand` shape. Defaults: `--context 512`, `--stride 256` (i.e. `L/2`, llama.cpp's default), `--mode sliding-window`, `--max-tokens 0`. + +Output must report `Perplexity`, `MeanNegativeLogLikelihood`, `ScoredTokens`, `WindowCount`, plus the effective context and stride — a perplexity figure without its token count and window geometry is not comparable to anything, which is the failure this harness exists to prevent. + +- [ ] **Step 2: Register the verb in `Program.cs`** + +```csharp +config.AddCommand("perplexity") + .WithDescription("Compute perplexity over a text corpus.") + .WithExample("perplexity", "QuantFactory/SmolLM-135M-GGUF", "--corpus", "wiki.test.raw", "--context", "512", "--stride", "256"); +``` + +- [ ] **Step 3: Verify end-to-end on a small real model** + +Run: `dotnet run --project src/DotLLM.Cli -- perplexity QuantFactory/SmolLM-135M-GGUF --corpus --context 256 --stride 128 --max-tokens 2048` +Expected: a finite perplexity in a plausible range for a 135M model on English prose (roughly 15–60), with `ScoredTokens` and `WindowCount` consistent with the tiling — `WindowCount == floor((tokens - context)/stride) + 1` and `ScoredTokens == WindowCount * stride`. + +- [ ] **Step 4: Commit** + +```bash +git add src/DotLLM.Cli/Commands/PerplexityCommand.cs src/DotLLM.Cli/Program.cs +git commit -m "feat(cli): dotllm perplexity verb (#231)" +``` + +--- + +### Task 9: llama.cpp comparability validation + +**Files:** +- Create: `tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs` + +**Interfaces:** +- Consumes: all of the above. +- Produces: the acceptance evidence for the spec's second verification requirement. + +Without this the word "comparable" is unearned, and the harness would give upstream false confidence on exactly the numerics-changing decisions it is meant to gate. + +- [ ] **Step 1: Produce the reference figure** + +Run llama.cpp's own perplexity on a fixed model + corpus, recording the exact build, model file, quantization, context and stride: + +```bash +llama-perplexity -m -f wiki.test.raw -c 512 +``` + +Record the reported perplexity, chunk count, and token count in the test file as constants with a comment naming the llama.cpp build hash. **Do not** paste a figure from a blog post — the model file and quantization must match ours exactly. + +- [ ] **Step 2: Write the test** + +`[SkippableFact]`, skipped when the corpus or GGUF is unavailable (follow the `DOTLLM_BITNET_GGUF` early-return pattern already used in the integration suite). Assert our perplexity is within a stated relative tolerance of the recorded llama.cpp figure, and that `ScoredTokens` matches llama.cpp's token count. + +Start with a 1% tolerance. If it fails, **do not widen the tolerance to make it pass** — a real discrepancy means the tiling or the scored-range definition differs, and that is the bug this task exists to catch. Investigate the window geometry first. + +- [ ] **Step 3: Run it** + +Run: `dotnet test tests/DotLLM.Tests.Integration --filter "FullyQualifiedName~PerplexityComparabilityTests"` +Expected: PASS, or a documented discrepancy with the geometry investigated. + +- [ ] **Step 4: Commit and open the PR** + +```bash +git add tests/DotLLM.Tests.Integration/Evaluation/PerplexityComparabilityTests.cs +git commit -m "test(eval): validate sliding-window perplexity against llama.cpp reference (#231)" +``` + +Then open the PR against upstream `main` referencing `Closes #231` and linking kkokosa/dotLLM#416. + +--- + +## Follow-up (separate, on `dev`) + +Migrate the ten existing per-test perplexity helpers onto this harness, gated on producing **numerically identical** results. Not part of this PR: those helpers exist only on `dev`, and this PR targets upstream `main`. File as its own issue once this lands. + +## Self-Review + +**Spec coverage:** contract (landed) · evaluator both modes (Tasks 3–5) · corpus streaming (Task 6) · adapter (Task 7) · CLI verb (Task 8) · comparability validation (Task 9) · never-loads-weights constraint (Task 7 adapter is borrowed-reference; evaluator signature takes a model) · streaming constraint (Task 6). Migration of existing helpers is explicitly deferred with a reason. + +**Placeholders:** none. Task 8's command body follows an existing in-repo pattern rather than inventing one, and its acceptance is a concrete arithmetic check on window geometry. + +**Type consistency:** `LogProb.OfTarget`, `PerplexityEvaluator.Evaluate`, `CorpusReader.StreamTokens`, `TransformerPerplexityModel` are used with identical signatures everywhere they appear. `TeacherForcedSinglePass`/`TeacherForcedGrowingPrefix` naming is fixed in Task 4. diff --git a/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md b/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md new file mode 100644 index 00000000..9b9e9804 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-perplexity-harness-design.md @@ -0,0 +1,110 @@ +# Perplexity harness — design + +**Issue:** jamesburton/dotLLM#231 · **Upstream driver:** kkokosa/dotLLM#416 · **Date:** 2026-07-30 + +## Problem + +Upstream #416's investigation log ends on a blocker: *"Every remaining lever changes numerics and +none of them can currently be evaluated."* Scale-granularity repacking (G=64/128, estimated +1.31–1.56×) and every other numerics-changing CPU lever are gated on an evaluation harness that +does not exist upstream. + +`upstream/main` has only `samples/DotLLM.Sample.Logprobs/Program.cs`. Our `dev` has perplexity +scoring in ten files — but entirely as **duplicated private per-test helpers**. `StableLogProb` is +copy-pasted verbatim; the same English corpus string appears in multiple files; two different +scoring strategies have diverged without a shared definition. None of it is reachable as a gate for +kernel work. + +So this is consolidation plus extension, not greenfield — but the consolidation half must be done +from `dev`, because that is where the helpers live. See Sequencing. + +## The one real axis + +The existing helpers diverged into two shapes for a single reason: **whether the backend's +`Forward` returns logits for every position or only the last one.** + +- CPU returns `[seqLen, vocab]`, so one teacher-forced prefill scores every next-token NLL — O(n). + (`BitNetAccuracyTests.Cpu_Perplexity_OnFixedPassage_IsSane`) +- CUDA returns only the final row, so each target needs its own growing-prefix re-prefill — O(n²), + which is why those helpers carry a stride to keep the sweep brisk. + (`CudaFlashPrefillForwardHarness.PrefillGrowingPrefixPerplexity`, and its twin in + `CudaG3PrefillForwardHarness`) + +Everything else about them is identical. Modelling that axis explicitly — `ReturnsAllRows` on +`IPerplexityModel` — is what lets one evaluator replace both without changing any number either +currently produces. + +## Components + +**`DotLLM.Core.Evaluation`** (abstractions only, so `main` and `dev` can both reference the +contract without dragging in the implementation): + +- `IPerplexityModel` — `VocabSize`, `MaxContextLength`, `ReturnsAllRows`, `Forward(tokens, positions)`. + Deliberately narrower than `IModel`: perplexity needs no sampling, no KV-cache lifetime + management, no streaming, and binding to the full interface would prevent scoring a bare backend + or a test double. +- `PerplexityMode` — `TeacherForced` | `SlidingWindow`. +- `PerplexityOptions(Mode, ContextLength, Stride, MaxTokens)`. +- `PerplexityResult(Perplexity, MeanNegativeLogLikelihood, ScoredTokens, WindowCount)`. + +**`PerplexityEvaluator`** — strategy selected from `ReturnsAllRows`, not from the caller. + +**Corpus handling** — streamed and tokenized in chunks. + +**CLI verb** — context length, stride, corpus path, token cap. + +## Two modes, two purposes + +`TeacherForced` preserves the in-tree "G1 precedent" methodology exactly. It is **ratio-oriented**: +the load-bearing signal is the OFF/ON perplexity ratio on identical tokens under a <1% gate, not +the absolute value. Not comparable to published figures, and must not be presented as if it were. + +`SlidingWindow` is new and **absolute-value oriented**: windows of `ContextLength` advanced by +`Stride`, scoring only tokens beyond the carried-over prefix so every scored token has full-length +context. Matches llama.cpp's `--perplexity` methodology so figures compare directly to published +numbers. + +## Memory constraint (from planned Track D) + +On Strix Halo's UMA, a large VRAM carve-out leaves host RAM scarce, and perplexity is the workload +most punished by it — a long sequence of full-context prefills rather than a single load. A harness +that mmaps weights host-side while the backend separately uploads them pays for the model twice +against an already-halved budget. + +Therefore, as a design constraint rather than a later optimisation: + +- **The evaluator never loads weights.** It takes an already-constructed `IPerplexityModel`. +- **The corpus is streamed and tokenized in chunks**, never materialized whole. + +Both cost nothing now and keep Track D a pure optimisation rather than a rewrite. + +## Verification + +1. **Consolidation is behaviour-preserving.** Migrated tests must produce *numerically identical* + results to their previous private helpers. A changed number means the consolidation altered + semantics — this is the primary regression gate, and it is the reason `TeacherForced` is + preserved verbatim rather than "improved" in passing. +2. **`SlidingWindow` is genuinely comparable.** Validated against a published llama.cpp perplexity + figure on matching model, corpus, context length and stride, within a stated tolerance. Without + this the word "comparable" is unearned, and the harness would give upstream false confidence on + exactly the numerics-changing decisions it is meant to gate. +3. **`ScoredTokens` is reported and checked.** A perplexity over a different token count is a + different measurement; cross-run comparison requires it to match. + +## Sequencing + +Built from `main` in a worktree (`issue/231-perplexity-harness`), PR targets upstream `main`, then +merges to `dev`. + +Because the ten duplicated helpers exist only on `dev`, the split is: + +- **On `main` / this PR:** the contract, the evaluator, corpus handling, CLI verb, and validation + of `SlidingWindow` against llama.cpp. Self-contained and upstream-contributable. +- **On `dev`, follow-up:** migrate the ten existing helpers onto the harness, gated on producing + identical numbers. + +## Out of scope + +Track D (iGPU optimisation of the harness under a large VRAM carve-out, including eliminating +host+device double-loading) is deliberately deferred until this and #233 land. The constraints +above exist so D is optimisation, not rework. diff --git a/src/DotLLM.Cli/Commands/PerplexityCommand.cs b/src/DotLLM.Cli/Commands/PerplexityCommand.cs new file mode 100644 index 00000000..e5c39783 --- /dev/null +++ b/src/DotLLM.Cli/Commands/PerplexityCommand.cs @@ -0,0 +1,245 @@ +using System.ComponentModel; +using System.Diagnostics; +using DotLLM.Core.Configuration; +using DotLLM.Core.Evaluation; +using DotLLM.Core.Models; +using DotLLM.Engine; +using DotLLM.Engine.Evaluation; +using DotLLM.Models.Architectures; +using DotLLM.Models.Evaluation; +using DotLLM.Models.Gguf; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace DotLLM.Cli.Commands; + +/// +/// Computes perplexity over a text corpus: load → stream-tokenize → score. +/// +/// +/// Defaults to with stride = context and +/// unscored prefix = context / 2 + 1 — non-overlapping chunks, each scoring the targets after +/// its midpoint. That reproduces llama.cpp's --perplexity methodology, so the reported figure +/// is directly comparable to published numbers for the same model, corpus and context. +/// Advance and scored span are separate knobs on purpose: llama.cpp advances by the whole +/// window yet scores only part of it, so its scored ranges have gaps. A single "stride" cannot +/// express that. +/// +internal sealed class PerplexityCommand : AsyncCommand +{ + /// + /// Delimiters accepted in a --tokens-file: whitespace plus the punctuation of a + /// JSON array, so a reference implementation's dump parses as-is. + /// + private static readonly char[] TokenIdSeparators = [' ', '\t', '\r', '\n', ',', '[', ']']; + + public sealed class Settings : CommandSettings + { + [CommandArgument(0, "")] + [Description("Path to a GGUF file or HuggingFace repo ID (e.g., QuantFactory/SmolLM-135M-GGUF).")] + public string Model { get; set; } = string.Empty; + + [CommandOption("--corpus|-f")] + [Description("Path to a UTF-8 text corpus (e.g. wiki.test.raw).")] + public string Corpus { get; set; } = string.Empty; + + [CommandOption("--context|-c")] + [Description("Context window in tokens. Clamped to the model's maximum sequence length.")] + [DefaultValue(512)] + public int Context { get; set; } = 512; + + [CommandOption("--stride")] + [Description("Tokens advanced between window starts. 0 selects the context length (non-overlapping chunks, llama.cpp's default).")] + [DefaultValue(0)] + public int Stride { get; set; } + + [CommandOption("--unscored-prefix")] + [Description("Leading tokens of each window used as context only. -1 selects context/2 + 1, which scores the same targets as llama.cpp.")] + [DefaultValue(-1)] + public int UnscoredPrefix { get; set; } = -1; + + [CommandOption("--max-tokens|-n")] + [Description("Cap on corpus tokens consumed. 0 = unbounded.")] + [DefaultValue(0)] + public int MaxTokens { get; set; } + + [CommandOption("--mode")] + [Description("Scoring mode: sliding-window (default, llama.cpp-comparable) or teacher-forced.")] + [DefaultValue("sliding-window")] + public string Mode { get; set; } = "sliding-window"; + + [CommandOption("--tokens-file")] + [Description("Read pre-tokenized whitespace-separated ids instead of tokenizing --corpus. Isolates scoring from tokenization when comparing against another implementation.")] + public string? TokensFile { get; set; } + + [CommandOption("--dump-tokens")] + [Description("Write the tokenized corpus ids to this path, whitespace-separated, then continue. Diagnostic.")] + public string? DumpTokens { get; set; } + + [CommandOption("--per-window")] + [Description("Print each window's perplexity. Use to localize a disagreement with another implementation to specific corpus content.")] + [DefaultValue(false)] + public bool PerWindow { get; set; } + + [CommandOption("--bos")] + [Description("Substitute BOS at the start of each window. Match the model's add_bos setting: llama.cpp only does this when the tokenizer requests it.")] + [DefaultValue(false)] + public bool Bos { get; set; } + + [CommandOption("--quant")] + [Description("Quantization to select when resolving a HuggingFace repo ID.")] + public string? Quant { get; set; } + + [CommandOption("--threads")] + [Description("Compute threads. 0 = auto.")] + [DefaultValue(0)] + public int Threads { get; set; } + } + + public override async Task ExecuteAsync(CommandContext context, Settings settings) + { + // --tokens-file supplies the token stream directly, so it replaces --corpus rather than + // supplementing it. Requiring both would defeat the flag's purpose: scoring a reference + // implementation's exact ids to separate a tokenizer difference from a scoring one. + if (settings.TokensFile is not null) + { + if (!File.Exists(settings.TokensFile)) + { + AnsiConsole.MarkupLine( + $"[red]Tokens file not found: {Markup.Escape(settings.TokensFile)}[/]"); + return 1; + } + } + else if (string.IsNullOrWhiteSpace(settings.Corpus)) + { + AnsiConsole.MarkupLine("[red]--corpus is required (or --tokens-file).[/]"); + return 1; + } + else if (!File.Exists(settings.Corpus)) + { + AnsiConsole.MarkupLine($"[red]Corpus not found: {Markup.Escape(settings.Corpus)}[/]"); + return 1; + } + + if (!TryParseMode(settings.Mode, out PerplexityMode mode)) + { + AnsiConsole.MarkupLine( + $"[red]Unknown --mode '{Markup.Escape(settings.Mode)}'. Expected 'sliding-window' or 'teacher-forced'.[/]"); + return 1; + } + + string? resolvedPath = GgufFileResolver.Resolve(settings.Model, settings.Quant); + if (resolvedPath is null) + return 1; + + using GgufFile gguf = GgufFile.Open(resolvedPath); + ModelConfig config = GgufModelConfigExtractor.Extract(gguf.Metadata); + var tokenizer = GgufBpeTokenizerFactory.Load(gguf.Metadata); + using TransformerModel model = TransformerModel.LoadFromGguf( + gguf, config, new ThreadingConfig(settings.Threads)); + + int effectiveContext = Math.Min(settings.Context, config.MaxSequenceLength); + // Defaults reproduce llama.cpp: non-overlapping chunks, scoring the second half of each. + int effectiveStride = settings.Stride > 0 ? settings.Stride : effectiveContext; + // context/2 + 1, not context/2: llama.cpp scores targets (n_ctx/2, n_ctx), leaving the token + // at n_ctx/2 as context only. See PerplexityOptions.LlamaCppDefault. + int effectivePrefix = settings.UnscoredPrefix >= 0 + ? settings.UnscoredPrefix + : Math.Min(effectiveContext - 1, Math.Max(1, effectiveContext / 2 + 1)); + + // Streamed, then buffered once: scoring needs random access across windows, but the file + // itself is never held in memory and the token list is bounded by --max-tokens. + var tokens = new List(); + if (settings.TokensFile is not null) + { + // Accept both bare whitespace-separated ids and the JSON-array form that reference + // tools print, so a dump can be pasted in without reformatting. + foreach (string part in File.ReadAllText(settings.TokensFile) + .Split(TokenIdSeparators, StringSplitOptions.RemoveEmptyEntries)) + { + tokens.Add(int.Parse(part)); + if (settings.MaxTokens > 0 && tokens.Count >= settings.MaxTokens) break; + } + } + else + { + using var reader = new StreamReader(settings.Corpus); + foreach (int id in CorpusReader.StreamTokens(reader, tokenizer, settings.MaxTokens)) + tokens.Add(id); + } + + if (tokens.Count < 2) + { + AnsiConsole.MarkupLine($"[red]Corpus tokenized to {tokens.Count} tokens; at least 2 are required.[/]"); + return 1; + } + + if (settings.DumpTokens is not null) + File.WriteAllText(settings.DumpTokens, string.Join(' ', tokens)); + + var perplexityModel = new TransformerPerplexityModel(model, deviceId: -1); + int bosTokenId = settings.Bos ? tokenizer.BosTokenId : -1; + var options = new PerplexityOptions( + mode, effectiveContext, effectiveStride, settings.MaxTokens, effectivePrefix, bosTokenId); + + var sw = Stopwatch.StartNew(); + PerplexityResult result; + try + { + PerplexityEvaluator.WindowObserver? observer = settings.PerWindow + ? (i, ppl, n) => Console.WriteLine($"window {i}: ppl={ppl:F6} scored={n}") + : null; + result = PerplexityEvaluator.Evaluate( + perplexityModel, + System.Runtime.InteropServices.CollectionsMarshal.AsSpan(tokens), + options, + observer); + } + catch (ArgumentException ex) + { + AnsiConsole.MarkupLine($"[red]{Markup.Escape(ex.Message)}[/]"); + return 1; + } + sw.Stop(); + + // Window geometry and scored-token count are reported alongside the figure deliberately: + // a perplexity without them is not comparable to anything. + var table = new Table().Border(TableBorder.Rounded); + table.AddColumn("Metric"); + table.AddColumn(new TableColumn("Value").RightAligned()); + // Printed as "PPL +/- err" in llama.cpp's own format so the two can be compared by eye. + // Without the error bar a reader has no way to tell a regression from sampling noise. + table.AddRow("Perplexity", $"{result.Perplexity:F4} +/- {result.StandardError:F5}"); + table.AddRow("Mean NLL (nats)", $"{result.MeanNegativeLogLikelihood:F6}"); + table.AddRow("Scored tokens", $"{result.ScoredTokens:N0}"); + table.AddRow("Windows", $"{result.WindowCount:N0}"); + table.AddRow("Mode", mode == PerplexityMode.SlidingWindow ? "sliding-window" : "teacher-forced"); + table.AddRow("Context", $"{effectiveContext:N0}"); + table.AddRow("Stride", $"{effectiveStride:N0}"); + table.AddRow("Unscored prefix", $"{effectivePrefix:N0}"); + table.AddRow("Corpus tokens", $"{tokens.Count:N0}"); + table.AddRow("Elapsed", $"{sw.Elapsed.TotalSeconds:F2} s"); + AnsiConsole.Write(table); + + await Task.CompletedTask; + return 0; + } + + private static bool TryParseMode(string value, out PerplexityMode mode) + { + switch (value.Trim().ToLowerInvariant()) + { + case "sliding-window": + case "sliding": + mode = PerplexityMode.SlidingWindow; + return true; + case "teacher-forced": + case "teacher": + mode = PerplexityMode.TeacherForced; + return true; + default: + mode = default; + return false; + } + } +} diff --git a/src/DotLLM.Cli/Program.cs b/src/DotLLM.Cli/Program.cs index be553e78..b7ad93e2 100644 --- a/src/DotLLM.Cli/Program.cs +++ b/src/DotLLM.Cli/Program.cs @@ -48,6 +48,10 @@ .WithDescription("Interactive multi-turn chat with a GGUF model.") .WithExample("chat", "QuantFactory/SmolLM-135M-GGUF", "--system", "You are a helpful assistant."); + config.AddCommand("perplexity") + .WithDescription("Compute perplexity over a text corpus.") + .WithExample("perplexity", "QuantFactory/SmolLM-135M-GGUF", "--corpus", "wiki.test.raw", "--context", "512", "--stride", "256"); + config.AddCommand("serve") .WithDescription("Launch API server with built-in web chat UI.") .WithExample("serve", "Qwen/Qwen3-0.6B-GGUF", "--port", "8080"); diff --git a/src/DotLLM.Core/Evaluation/IPerplexityModel.cs b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs new file mode 100644 index 00000000..71e57141 --- /dev/null +++ b/src/DotLLM.Core/Evaluation/IPerplexityModel.cs @@ -0,0 +1,61 @@ +using DotLLM.Core.Tensors; + +namespace DotLLM.Core.Evaluation; + +/// +/// The minimal model surface perplexity scoring needs: a teacher-forced forward pass over a +/// token window, plus enough metadata to interpret the logits it returns. +/// +/// +/// Deliberately narrower than IModel. Perplexity needs neither sampling, KV-cache +/// lifetime management, nor streaming, and binding the evaluator to the full model interface +/// would prevent scoring a bare backend or a test double. +/// The evaluator never loads weights. Implementations are handed an +/// already-constructed model, so a caller that has the weights resident on a device is never +/// forced into a second host-side copy. This matters on unified-memory parts (e.g. Strix Halo) +/// where a large VRAM carve-out leaves host RAM scarce, and perplexity — a long sequence of +/// full-context prefills rather than a single load — is the workload most punished by paying +/// for the model twice. +/// +public interface IPerplexityModel +{ + /// Vocabulary size; the row length of the returned logits. + int VocabSize { get; } + + /// + /// Maximum token window a single call accepts. Sliding-window scoring + /// never requests a window larger than this. + /// + int MaxContextLength { get; } + + /// + /// when returns logits for every position + /// (shape [seqLen, VocabSize]); when only the final row is + /// returned (shape [1, VocabSize] or [VocabSize]). + /// + /// + /// This is the single axis that decides scoring cost, and the reason the existing per-test + /// helpers diverged into two shapes. All-rows backends score a window in one forward pass + /// (O(n)); last-row-only backends must re-prefill each growing prefix (O(n^2)). The evaluator + /// selects the strategy from this flag rather than the caller hard-coding one. + /// + bool ReturnsAllRows { get; } + + /// + /// Runs a teacher-forced forward pass over . + /// + /// Input token ids for this window. + /// + /// Position ids, one per token, passed explicitly rather than derived so the caller controls + /// them. + /// Do not assume they are absolute corpus offsets. Sliding-window scoring passes + /// window-relative positions restarting at 0 for every window, because llama.cpp + /// evaluates each chunk as an independent sequence and matching it is the point of that mode. + /// It is also what lets a corpus longer than be scored at all — + /// absolute positions would run past the model's limit on the second window. + /// + /// + /// Logits, owned by the caller. Row layout is governed by . + /// + ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions); +} diff --git a/src/DotLLM.Core/Evaluation/PerplexityResult.cs b/src/DotLLM.Core/Evaluation/PerplexityResult.cs new file mode 100644 index 00000000..bec785e1 --- /dev/null +++ b/src/DotLLM.Core/Evaluation/PerplexityResult.cs @@ -0,0 +1,134 @@ +namespace DotLLM.Core.Evaluation; + +/// +/// Scoring strategy. Chosen by the caller; the evaluator picks the execution path from +/// . +/// +public enum PerplexityMode +{ + /// + /// Teacher-forced scoring over a single window — the first + /// tokens, clamped to the model's maximum — scored from one forward pass where the backend + /// permits it. The established in-tree methodology (the "G1 precedent" referenced by the CUDA + /// prefill harnesses); preserved so existing quality gates keep their meaning after consolidation. + /// + /// + /// and + /// are ignored in this mode: it scores one window and stops. That is deliberate — the mode + /// exists to reproduce the pre-existing harnesses bit for bit, and giving it window geometry + /// would change the numbers those gates were calibrated against. Use + /// to walk a corpus. + /// Ratio-oriented: the load-bearing signal is the OFF/ON perplexity ratio on identical + /// tokens, not the absolute value. Not comparable to published figures. + /// + TeacherForced, + + /// + /// Sliding-window scoring. The corpus is walked in windows of + /// advanced by + /// , and each window scores only the targets beyond its + /// , so every scored token carries that many + /// tokens of context. + /// + /// + /// Absolute-value oriented: comparable to published llama.cpp figures when model, corpus, + /// context, stride and unscored prefix all match. See + /// . + /// + SlidingWindow, +} + +/// Configuration for a perplexity run. +/// Scoring strategy. +/// +/// Window size L in tokens. Clamped to . +/// +/// +/// Tokens advanced between window starts. Stride == ContextLength gives non-overlapping +/// windows; a smaller value overlaps them. +/// +/// +/// Upper bound on corpus tokens consumed; 0 means unbounded. Bounds runtime on large +/// corpora without truncating the corpus file itself. +/// +/// +/// Leading tokens of each window used only as context and never scored. -1 derives +/// ContextLength - Stride, which makes the scored ranges tile the corpus contiguously. +/// +/// +/// When non-negative, each window's first token is replaced by this id. llama.cpp does this for +/// every chunk, since each is evaluated as a fresh sequence; the substituted slot lies inside the +/// unscored prefix, so no scored target changes. -1 disables the substitution. +/// +/// +/// Advance and scored span are independent. A single "stride" cannot express +/// llama.cpp's scheme: it advances by the full window yet scores only the second half, so its +/// scored ranges have gaps. Collapsing the two into one knob silently produces a different token +/// set — the same count, scored over different tokens — and therefore a figure that looks +/// comparable but is not. +/// Scored targets in a window starting at s are the absolute indices +/// [s + UnscoredPrefix, s + ContextLength). +/// +public readonly record struct PerplexityOptions( + PerplexityMode Mode, + int ContextLength, + int Stride, + int MaxTokens = 0, + int UnscoredPrefix = -1, + int BosTokenId = -1) +{ + /// + /// Options reproducing llama.cpp's --perplexity defaults for a given context: + /// non-overlapping chunks of , scoring the second half of each. + /// + /// + /// This is the configuration whose output is directly comparable to published llama.cpp + /// figures. Verified against llama.cpp build 8683 (d0a6dfeb2). + /// The unscored prefix is contextLength / 2 + 1, not contextLength / 2. + /// llama.cpp sets first = n_ctx/2 and then accumulates count += n_ctx - first - 1, + /// scoring targets [first + 1, n_ctx) — the token at index first is context, never + /// a target. Scoring it too yields n_ctx/2 targets where llama.cpp has + /// n_ctx/2 - 1, which is a different measurement wearing the same name. + /// + /// Window size. + /// Corpus token cap; 0 for unbounded. + /// + /// BOS id to substitute at the start of each chunk, mirroring llama.cpp; -1 disables. + /// + public static PerplexityOptions LlamaCppDefault(int contextLength, int maxTokens = 0, int bosTokenId = -1) => + new(PerplexityMode.SlidingWindow, contextLength, Stride: contextLength, maxTokens, + UnscoredPrefix: contextLength / 2 + 1, BosTokenId: bosTokenId); +} + +/// Outcome of a perplexity run. +/// +/// exp(MeanNegativeLogLikelihood) — the headline figure. +/// +/// +/// Mean NLL in nats over all scored tokens. Reported alongside perplexity because differences +/// between near-identical runs are easier to read here than through the exponential. +/// +/// +/// Number of tokens that contributed. Comparisons across runs are meaningful only when this +/// matches: a perplexity computed over a different token count is a different measurement. +/// +/// Number of forward windows evaluated. +/// +/// One standard error of , from the sample variance of the per-token NLL: +/// sqrt(Var(nll) / (ScoredTokens - 1)) * Perplexity. 0 when fewer than two tokens +/// were scored, or when the variance is non-positive through rounding. +/// +/// +/// Read the error bar before comparing two perplexities. It is the difference between +/// a real regression and sampling noise, and on short corpora it is far wider than intuition +/// suggests: on a 2,286-token corpus this model's figure carries roughly ±6.5%, so a 3% +/// "discrepancy" against another implementation is not evidence of anything. The same model and +/// corpus family at wikitext-2 scale (150,195 scored tokens) narrows that to ±0.8%. +/// Matches llama.cpp's +/- figure, so the two are directly comparable. +/// +public readonly record struct PerplexityResult( + double Perplexity, + double MeanNegativeLogLikelihood, + int ScoredTokens, + int WindowCount, + double StandardError = 0); diff --git a/src/DotLLM.Engine/Evaluation/CorpusReader.cs b/src/DotLLM.Engine/Evaluation/CorpusReader.cs new file mode 100644 index 00000000..5418ee27 --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/CorpusReader.cs @@ -0,0 +1,104 @@ +using System.Text; +using DotLLM.Tokenizers; + +namespace DotLLM.Engine.Evaluation; + +/// Streams a text corpus into tokens without materializing the whole file or token array. +/// +/// Streaming is a design constraint rather than an optimisation. On unified-memory parts a large +/// VRAM carve-out leaves host RAM scarce, and a standard perplexity corpus tokenizes to hundreds of +/// thousands of ints — held alongside the weights, that is exactly the pressure this harness must +/// not add. +/// +public static class CorpusReader +{ + /// + /// Reads in character chunks, tokenizes each chunk, and yields token + /// ids in order, stopping after (0 = unbounded). + /// + /// + /// Chunks are cut immediately before the last run of whitespace, so a token is never split + /// across a boundary; the remainder is carried into the next chunk, and the final carry is + /// flushed whole. Verified against whole-file tokenization on wikitext-2 (1.29 MB, 301,948 + /// tokens) at chunk sizes from 997 to 65536 — identical streams, so ~1,290 boundaries in the + /// smallest case land in awkward places without changing a single id. + /// Whole runs, not single characters. Cutting at the last whitespace character + /// can fall inside a run of them, and a GPT-2-style pre-tokenizer treats a whitespace run as one + /// unit — splitting it yields a different token stream than tokenizing the file in one pass, + /// which is precisely the silent divergence this harness exists to rule out. + /// Known limitation: text containing no whitespace at all accumulates in the carry + /// buffer until some arrives, or until the corpus ends. Flushing at an arbitrary character + /// instead would bound the memory but change the token stream, so correctness wins here. Ordinary + /// prose corpora are unaffected; a whitespace-free corpus (minified JSON, unsegmented CJK) is + /// effectively read whole. + /// + /// Corpus source. + /// Tokenizer whose vocabulary the ids belong to. + /// Upper bound on emitted tokens; 0 for unbounded. + /// Characters read per chunk. + public static IEnumerable StreamTokens( + TextReader reader, ITokenizer tokenizer, int maxTokens = 0, int charChunkSize = 65536) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(tokenizer); + ArgumentOutOfRangeException.ThrowIfLessThan(charChunkSize, 1); + + var buffer = new char[charChunkSize]; + var carry = new StringBuilder(); + int emitted = 0; + + while (true) + { + int read = reader.Read(buffer, 0, buffer.Length); + if (read == 0) break; + + carry.Append(buffer, 0, read); + string pending = carry.ToString(); + + int cut = LastWhitespaceRunStart(pending); + if (cut <= 0) continue; // no safe split point yet; keep accumulating + + // The separating whitespace is carried INTO the next chunk, not dropped. GPT-2-style BPE + // encodes a leading space as part of the following token, so dropping it silently + // changes the token stream — and therefore the perplexity — versus tokenizing the + // corpus in one pass. + string ready = pending[..cut]; + carry.Clear(); + carry.Append(pending[cut..]); + + foreach (int id in tokenizer.Encode(ready)) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + + if (carry.Length > 0) + { + foreach (int id in tokenizer.Encode(carry.ToString())) + { + yield return id; + if (maxTokens > 0 && ++emitted >= maxTokens) yield break; + } + } + } + + /// + /// Index of the first character of the last whitespace run in , or + /// -1 when it contains no whitespace. + /// + /// + /// Returning the run's START rather than the last whitespace character is what keeps the run + /// intact: everything from it onwards moves into the carry, so no pre-token that spans a + /// whitespace run is ever cut in half. + /// + private static int LastWhitespaceRunStart(string text) + { + int i = text.Length - 1; + while (i >= 0 && !char.IsWhiteSpace(text[i])) i--; + if (i < 0) return -1; + + while (i > 0 && char.IsWhiteSpace(text[i - 1])) i--; + return i; + } +} diff --git a/src/DotLLM.Engine/Evaluation/LogProb.cs b/src/DotLLM.Engine/Evaluation/LogProb.cs new file mode 100644 index 00000000..a70b764e --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/LogProb.cs @@ -0,0 +1,33 @@ +namespace DotLLM.Engine.Evaluation; + +/// Numerically stable log-softmax over a single row of logits. +public static class LogProb +{ + /// + /// Returns log P(target) in nats under a softmax over . + /// + /// + /// Uses the max-shift identity log softmax(x)_t = (x_t - m) - log sum_j exp(x_j - m) + /// with m = max(x), so no exp argument is ever positive and overflow is + /// impossible. Accumulates in : a vocab of 128k float32 terms loses + /// meaningful precision in float32, and perplexity differences between near-identical runs + /// are exactly what this harness exists to resolve. + /// + /// One row of unnormalized scores. + /// Index whose log-probability is returned. + public static double OfTarget(ReadOnlySpan logits, int target) + { + if ((uint)target >= (uint)logits.Length) + throw new ArgumentOutOfRangeException(nameof(target)); + + float max = logits[0]; + for (int j = 1; j < logits.Length; j++) + if (logits[j] > max) max = logits[j]; + + double sumExp = 0; + for (int j = 0; j < logits.Length; j++) + sumExp += Math.Exp(logits[j] - max); + + return (logits[target] - max) - Math.Log(sumExp); + } +} diff --git a/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs new file mode 100644 index 00000000..cb03d34d --- /dev/null +++ b/src/DotLLM.Engine/Evaluation/PerplexityEvaluator.cs @@ -0,0 +1,242 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Engine.Evaluation; + +/// +/// Computes perplexity over a token sequence using an . +/// +/// +/// The evaluator never loads weights — callers pass an already-constructed model. On unified-memory +/// parts a large VRAM carve-out leaves host RAM scarce, and perplexity (a long run of full-context +/// prefills) is the workload most punished by holding a second host-side copy of the weights. +/// +public static class PerplexityEvaluator +{ + /// + /// Per-window callback. Intended for diagnosis — differencing per-window figures against + /// another implementation localizes a disagreement to specific corpus content, which an + /// aggregate figure cannot. + /// + /// Zero-based index of the window just scored. + /// Perplexity over that window's scored targets alone. + /// Number of targets scored in that window. + public delegate void WindowObserver(int windowIndex, double windowPerplexity, int scoredInWindow); + + /// Scores and returns the aggregate result. + /// An already-constructed model. Not owned; not disposed here. + /// Token ids to score. + /// Mode and window geometry. + /// Optional per-window diagnostic callback. + public static PerplexityResult Evaluate( + IPerplexityModel model, ReadOnlySpan tokens, PerplexityOptions options, + WindowObserver? onWindow = null) + { + ArgumentNullException.ThrowIfNull(model); + if (tokens.Length < 2) + throw new ArgumentException("At least two tokens are required to score one target.", nameof(tokens)); + + int context = Math.Min(options.ContextLength, model.MaxContextLength); + if (context < 2) + throw new ArgumentException("Context length must be at least 2.", nameof(options)); + + return options.Mode switch + { + PerplexityMode.TeacherForced => EvaluateTeacherForced(model, tokens, context), + PerplexityMode.SlidingWindow => EvaluateSlidingWindow( + model, tokens, context, options.Stride, options.UnscoredPrefix, options.BosTokenId, onWindow), + _ => throw new NotSupportedException($"Unknown mode {options.Mode}."), + }; + } + + /// + /// Streaming mean and variance of the per-token NLL, by Welford's algorithm. + /// + /// + /// Deliberately not the textbook E[x²] - E[x]²: the per-token NLL of a converged model is + /// tightly clustered about a mean of a few nats, so the two terms agree to most of their + /// significant digits and their difference is mostly rounding error. On a constant NLL that form + /// returns ~1e-6 where the true variance is 0 — small in absolute terms, but it is the whole + /// signal. Welford accumulates the deviation directly and stays exact there, while agreeing with + /// the closed form (and so with llama.cpp's figure) everywhere else. + /// + private struct NllAccumulator + { + private double _mean; + private double _sumSquaredDeviation; + + public int Count { get; private set; } + + public double Mean => _mean; + + public void Add(double value) + { + Count++; + double delta = value - _mean; + _mean += delta / Count; + _sumSquaredDeviation += delta * (value - _mean); + } + + /// + /// One standard error of exp(Mean): the standard error of the mean NLL, carried + /// through exp by the delta method. Matches llama.cpp's +/- figure. + /// + public readonly double StandardErrorOfPerplexity(double perplexity) + { + if (Count < 2) + return 0; + double variance = _sumSquaredDeviation / Count; + return variance > 0 ? Math.Sqrt(variance / (Count - 1)) * perplexity : 0; + } + } + + /// Builds the result from the accumulated per-token NLL statistics. + /// Accumulated per-token NLL statistics. + /// Number of forward windows evaluated. + private static PerplexityResult Summarize(in NllAccumulator nll, int windows) + { + double perplexity = Math.Exp(nll.Mean); + return new PerplexityResult( + perplexity, nll.Mean, nll.Count, windows, nll.StandardErrorOfPerplexity(perplexity)); + } + + private static PerplexityResult EvaluateTeacherForced( + IPerplexityModel model, ReadOnlySpan tokens, int context) + => model.ReturnsAllRows + ? TeacherForcedSinglePass(model, tokens, context) + : TeacherForcedGrowingPrefix(model, tokens, context); + + // Backend returns only the final row, so each target needs its own prefill over the growing + // prefix. O(n^2) in forward passes — unavoidable, and the reason the CUDA harnesses that + // originated this methodology carry a stride. + private static unsafe PerplexityResult TeacherForcedGrowingPrefix( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + int vocab = model.VocabSize; + var positions = new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + var accumulator = new NllAccumulator(); + for (int prefix = 1; prefix < length; prefix++) + { + using ITensor logits = model.Forward(tokens[..prefix], positions.AsSpan(0, prefix)); + var row = new ReadOnlySpan((void*)logits.DataPointer, vocab); + accumulator.Add(-LogProb.OfTarget(row, tokens[prefix])); + } + + return Summarize(accumulator, windows: accumulator.Count); + } + + // Window w starts at w*Stride and covers [start, start + L); it scores the absolute targets + // [start + prefix, start + L), where `prefix` tokens serve as context only. + // + // Advance (Stride) and scored span (L - prefix) are INDEPENDENT. llama.cpp advances by the + // full window yet scores only its second half, so its scored ranges have gaps; a scheme with + // one knob cannot express that, and collapsing them yields the same scored-token *count* over + // a different token *set* — a figure that looks comparable and is not. Verified the hard way + // against llama.cpp build 8683: contiguous tiling gave 24.88 where llama.cpp gave 24.01. + // + // Targets before the first window's scored range are never scored; llama.cpp skips them too, + // and "fixing" that would break the comparability this mode exists to provide. + private static unsafe PerplexityResult EvaluateSlidingWindow( + IPerplexityModel model, ReadOnlySpan tokens, int context, int stride, int unscoredPrefix, + int bosTokenId, WindowObserver? onWindow) + { + if (stride < 1 || stride > context) + throw new ArgumentException( + $"Stride must be in [1, {context}] for a context of {context}.", nameof(stride)); + + int prefix = unscoredPrefix >= 0 ? unscoredPrefix : context - stride; + if (prefix < 1 || prefix >= context) + throw new ArgumentException( + $"Unscored prefix must be in [1, {context - 1}] for a context of {context}; " + + "each scored token needs at least one token of context, and at least one token must be scored.", + nameof(unscoredPrefix)); + + if (!model.ReturnsAllRows) + throw new NotSupportedException( + "Sliding-window mode requires a backend that returns all rows. Use PerplexityMode.TeacherForced " + + "for last-row-only backends — re-prefilling per target inside a window would be O(n^2) and is " + + "already what the growing-prefix path does."); + + int vocab = model.VocabSize; + var positions = new int[context]; + var accumulator = new NllAccumulator(); + int windows = 0; + + // Positions restart at 0 for every window: each is an independent sequence, exactly as + // llama.cpp evaluates each chunk. This is also what lets a corpus longer than the model's + // max sequence length be scored at all. + for (int i = 0; i < context; i++) positions[i] = i; + + // When a BOS id is supplied, each window's first token is replaced by it, mirroring + // llama.cpp's perplexity: every chunk is a fresh sequence and is given a sequence start. + // The substituted slot sits inside the unscored prefix, so no scored target is altered. + int[]? windowBuffer = bosTokenId >= 0 ? new int[context] : null; + + for (int start = 0; start + context <= tokens.Length; start += stride) + { + ReadOnlySpan window; + if (windowBuffer is null) + { + window = tokens.Slice(start, context); + } + else + { + tokens.Slice(start, context).CopyTo(windowBuffer); + windowBuffer[0] = bosTokenId; + window = windowBuffer; + } + + using ITensor logits = model.Forward(window, positions); + windows++; + + double windowNll = 0; + int windowScored = 0; + + // Absolute targets [start + prefix, start + context); row for target t is t-start-1. + for (int t = start + prefix; t < start + context; t++) + { + int row = t - start - 1; + var span = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)row * vocab * sizeof(float)), vocab); + double nll = -LogProb.OfTarget(span, tokens[t]); + windowNll += nll; + windowScored++; + accumulator.Add(nll); + } + + onWindow?.Invoke(windows - 1, Math.Exp(windowNll / windowScored), windowScored); + } + + if (accumulator.Count == 0) + throw new ArgumentException( + $"Corpus of {tokens.Length} tokens is shorter than one context window of {context}.", nameof(tokens)); + + return Summarize(accumulator, windows); + } + + // Backend returns every row, so one forward pass scores every target: row i predicts token i+1. + private static unsafe PerplexityResult TeacherForcedSinglePass( + IPerplexityModel model, ReadOnlySpan tokens, int context) + { + int length = Math.Min(tokens.Length, context); + Span positions = length <= 512 ? stackalloc int[length] : new int[length]; + for (int i = 0; i < length; i++) positions[i] = i; + + var accumulator = new NllAccumulator(); + + using ITensor logits = model.Forward(tokens[..length], positions); + int vocab = model.VocabSize; + // Row i predicts token i+1, so the final row has no target within the window. + for (int i = 0; i < length - 1; i++) + { + var row = new ReadOnlySpan( + (void*)(logits.DataPointer + (nint)i * vocab * sizeof(float)), vocab); + accumulator.Add(-LogProb.OfTarget(row, tokens[i + 1])); + } + + return Summarize(accumulator, windows: 1); + } +} diff --git a/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs b/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs new file mode 100644 index 00000000..f0205a37 --- /dev/null +++ b/src/DotLLM.Models/Evaluation/TransformerPerplexityModel.cs @@ -0,0 +1,43 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; +using DotLLM.Models.Architectures; + +namespace DotLLM.Models.Evaluation; + +/// Adapts to . +/// +/// Holds a borrowed reference: the adapter does not own the model and does not dispose it, so the +/// caller keeps a single resident copy of the weights. This is the whole point of the evaluator +/// taking a constructed model rather than a path — on a unified-memory part with a large VRAM +/// carve-out, a second host-side copy is the difference between running and not. +/// +public sealed class TransformerPerplexityModel : IPerplexityModel +{ + private readonly TransformerModel _model; + private readonly int _deviceId; + + /// An already-loaded model. Not owned; not disposed by this adapter. + /// Device for the forward pass; -1 is CPU. + public TransformerPerplexityModel(TransformerModel model, int deviceId = -1) + { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _deviceId = deviceId; + } + + /// + public int VocabSize => _model.Config.VocabSize; + + /// + public int MaxContextLength => _model.Config.MaxSequenceLength; + + /// + /// + /// is + /// documented as returning logits of shape [seqLen, vocab_size] for all input positions. + /// + public bool ReturnsAllRows => true; + + /// + public ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + => _model.Forward(tokens, positions, _deviceId); +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs new file mode 100644 index 00000000..5428f415 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/CorpusReaderTests.cs @@ -0,0 +1,103 @@ +using DotLLM.Engine.Evaluation; +using DotLLM.Tokenizers; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class CorpusReaderTests +{ + // One token per whitespace-separated word; ids are word lengths, so order is checkable. + private sealed class WordTokenizer : ITokenizer + { + public int[] Encode(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(w => w.Length).ToArray(); + + public string Decode(ReadOnlySpan tokenIds) => throw new NotSupportedException(); + public string DecodeToken(int tokenId) => throw new NotSupportedException(); + public int CountTokens(string text) => Encode(text).Length; + public int VocabSize => 1024; + public int BosTokenId => 0; + public int EosTokenId => 1; + } + + [Fact] + public void StreamTokens_ProducesTokensInOrder() + { + using var reader = new StringReader("a bb ccc dddd"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer()).ToArray(); + Assert.Equal([1, 2, 3, 4], tokens); + } + + [Fact] + public void StreamTokens_HonoursMaxTokens() + { + using var reader = new StringReader("a bb ccc dddd eeeee"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 3).ToArray(); + Assert.Equal([1, 2, 3], tokens); + } + + [Fact] + public void StreamTokens_DoesNotSplitTokensAcrossChunkBoundaries() + { + // A tiny chunk size forces the boundary case: "ccc" must not become "c" + "cc". + using var reader = new StringReader("a bb ccc dddd eeeee ffffff"); + var tokens = CorpusReader.StreamTokens(reader, new WordTokenizer(), maxTokens: 0, charChunkSize: 4).ToArray(); + Assert.Equal([1, 2, 3, 4, 5, 6], tokens); + } + + /// Records the exact strings handed to the tokenizer, so cut points are assertable. + private sealed class RecordingTokenizer : ITokenizer + { + public List Segments { get; } = []; + + public int[] Encode(string text) + { + Segments.Add(text); + return [text.Length]; + } + + public string Decode(ReadOnlySpan tokenIds) => throw new NotSupportedException(); + public string DecodeToken(int tokenId) => throw new NotSupportedException(); + public int CountTokens(string text) => 1; + public int VocabSize => 1024; + public int BosTokenId => 0; + public int EosTokenId => 1; + } + + [Theory] + [InlineData("aa bb")] // run of spaces + [InlineData("aa \n\t bb")] // mixed whitespace run + [InlineData("aa\n\n\nbb")] // run of newlines, no spaces at all + [InlineData("aa\r\nbb")] // CRLF + public void StreamTokens_NeverCutsInsideAWhitespaceRun(string text) + { + // A GPT-2-style pre-tokenizer treats a whitespace run as one unit. Cutting inside one makes + // the streamed token stream differ from tokenizing the file in a single pass — a silent + // divergence that would invalidate every comparison this harness exists to support. + // Cutting on the last whitespace *character* rather than the run's start does exactly that, + // and a corpus of newlines with no spaces is not cut at all. + var tokenizer = new RecordingTokenizer(); + using var reader = new StringReader(text); + _ = CorpusReader.StreamTokens(reader, tokenizer, maxTokens: 0, charChunkSize: 4).ToArray(); + + Assert.Equal(text, string.Concat(tokenizer.Segments)); + Assert.All(tokenizer.Segments, s => Assert.False( + s.Length > 0 && char.IsWhiteSpace(s[^1]), + $"segment '{s.Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t")}' " + + "ends in whitespace, so a whitespace run was cut in half")); + } + + [Fact] + public void StreamTokens_WhitespaceFreeInputIsEmittedWhole() + { + // Documented limitation: with no whitespace there is no safe cut, so the carry accumulates + // and the corpus is effectively read whole. Asserted so the behaviour is a stated trade + // rather than a surprise. + var tokenizer = new RecordingTokenizer(); + using var reader = new StringReader("abcdefghijklmnop"); + _ = CorpusReader.StreamTokens(reader, tokenizer, maxTokens: 0, charChunkSize: 4).ToArray(); + + Assert.Single(tokenizer.Segments); + Assert.Equal("abcdefghijklmnop", tokenizer.Segments[0]); + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs b/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs new file mode 100644 index 00000000..ea6b1399 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/FakePerplexityModel.cs @@ -0,0 +1,57 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Core.Tensors; + +namespace DotLLM.Tests.Unit.Evaluation; + +/// +/// Deterministic for evaluator tests. Records every window it is +/// asked to score so tests can assert on window tiling, not just on the resulting number. +/// +internal sealed class FakePerplexityModel : IPerplexityModel, IDisposable +{ + private readonly Func _rowFactory; // (absolutePosition, vocabSize) => logits + private readonly List _forwardCalls = []; + private readonly List _issued = []; + + public FakePerplexityModel( + int vocabSize, int maxContextLength, bool returnsAllRows, + Func rowFactory) + { + VocabSize = vocabSize; + MaxContextLength = maxContextLength; + ReturnsAllRows = returnsAllRows; + _rowFactory = rowFactory; + } + + public int VocabSize { get; } + public int MaxContextLength { get; } + public bool ReturnsAllRows { get; } + + /// Token windows passed to , in call order. + public IReadOnlyList ForwardCalls => _forwardCalls; + + public unsafe ITensor Forward(ReadOnlySpan tokens, ReadOnlySpan positions) + { + _forwardCalls.Add(tokens.ToArray()); + + int rows = ReturnsAllRows ? tokens.Length : 1; + int firstRow = ReturnsAllRows ? 0 : tokens.Length - 1; + var tensor = UnmanagedTensor.Allocate(new TensorShape(rows, VocabSize), DType.Float32); + + var dest = new Span((void*)tensor.DataPointer, rows * VocabSize); + for (int r = 0; r < rows; r++) + _rowFactory(positions[firstRow + r], VocabSize).CopyTo(dest[(r * VocabSize)..]); + + _issued.Add(tensor); + return tensor; + } + + /// Uniform logits: every target scores exactly -log(vocabSize). + public static Func Uniform => (_, vocab) => new float[vocab]; + + public void Dispose() + { + foreach (var t in _issued) t.Dispose(); + _issued.Clear(); + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs new file mode 100644 index 00000000..9366e03f --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/LogProbTests.cs @@ -0,0 +1,43 @@ +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class LogProbTests +{ + [Fact] + public void OfTarget_UniformLogits_IsNegativeLogVocab() + { + // Four equal logits => p = 1/4 for each => log p = -log 4. + var logits = new float[] { 2.5f, 2.5f, 2.5f, 2.5f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.Equal(-Math.Log(4.0), actual, 12); + } + + [Fact] + public void OfTarget_IsShiftInvariant() + { + var a = new float[] { 1f, 2f, 3f }; + var b = new float[] { 1001f, 1002f, 1003f }; + Assert.Equal(LogProb.OfTarget(a, 1), LogProb.OfTarget(b, 1), 12); + } + + [Fact] + public void OfTarget_LargeLogits_DoesNotOverflow() + { + // Naive exp() would overflow to infinity here; the max-shift must prevent it. + var logits = new float[] { 800f, 900f, 1000f }; + double actual = LogProb.OfTarget(logits, target: 2); + Assert.True(double.IsFinite(actual)); + Assert.Equal(0.0, actual, 6); // target dominates => p ~ 1 => log p ~ 0 + } + + [Fact] + public void OfTarget_SumOfProbabilitiesIsOne() + { + var logits = new float[] { -1.5f, 0.25f, 3f, 0.5f }; + double sum = 0; + for (int i = 0; i < logits.Length; i++) sum += Math.Exp(LogProb.OfTarget(logits, i)); + Assert.Equal(1.0, sum, 10); + } +} diff --git a/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs new file mode 100644 index 00000000..ba0126d5 --- /dev/null +++ b/tests/DotLLM.Tests.Unit/Evaluation/PerplexityEvaluatorTests.cs @@ -0,0 +1,285 @@ +using DotLLM.Core.Evaluation; +using DotLLM.Engine.Evaluation; +using Xunit; + +namespace DotLLM.Tests.Unit.Evaluation; + +public sealed class PerplexityEvaluatorTests +{ + // Vocab must exceed every token id used as a scoring target. + private const int Vocab = 64; + private static readonly int[] Tokens = Enumerable.Range(0, 32).ToArray(); + + [Fact] + public void TeacherForced_AllRowsBackend_UniformLogitsGivesVocabSizePerplexity() + { + // Uniform logits => P(any target) = 1/vocab => perplexity == vocab, exactly. + using var model = new FakePerplexityModel( + vocabSize: Vocab, maxContextLength: 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, ContextLength: 32, Stride: 32)); + + Assert.Equal(Vocab, result.Perplexity, 9); + Assert.Equal(31, result.ScoredTokens); // n-1 targets from one pass + } + + [Fact] + public void TeacherForced_AllRowsBackend_UsesASingleForwardPass() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // The whole point of ReturnsAllRows: one pass scores every target. + Assert.Single(model.ForwardCalls); + Assert.Equal(32, model.ForwardCalls[0].Length); + } + + [Fact] + public void MeanNll_AndPerplexity_AreConsistent() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + Assert.Equal(result.Perplexity, Math.Exp(result.MeanNegativeLogLikelihood), 9); + } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_MatchesAllRowsBackendExactly() + { + // Position-dependent but deterministic logits, so a wrong row/position mapping shows up. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + for (int j = 0; j < vocab; j++) row[j] = (float)Math.Sin((position + 1) * (j + 1) * 0.37); + return row; + } + + using var allRows = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + using var lastRow = new FakePerplexityModel(Vocab, 64, returnsAllRows: false, Rows); + var options = new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32); + + var a = PerplexityEvaluator.Evaluate(allRows, Tokens, options); + var b = PerplexityEvaluator.Evaluate(lastRow, Tokens, options); + + Assert.Equal(a.Perplexity, b.Perplexity, 9); + Assert.Equal(a.ScoredTokens, b.ScoredTokens); + } + + [Fact] + public void TeacherForced_LastRowOnlyBackend_ReprefixesGrowingWindows() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: false, FakePerplexityModel.Uniform); + + PerplexityEvaluator.Evaluate( + model, Tokens, new PerplexityOptions(PerplexityMode.TeacherForced, 32, 32)); + + // One forward per scored target, each one token longer than the last. + Assert.Equal(31, model.ForwardCalls.Count); + for (int i = 0; i < model.ForwardCalls.Count; i++) + Assert.Equal(i + 1, model.ForwardCalls[i].Length); + } + + [Fact] + public void SlidingWindow_TilesScoredTokensWithoutGapsOrOverlap() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16, S=8 => windows start at 0, 8, 16, 24; each scores its last 8 targets. + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 8)); + + Assert.Equal(4, result.WindowCount); + Assert.Equal(32, result.ScoredTokens); // 4 windows x 8 targets + Assert.Equal(Vocab, result.Perplexity, 9); // uniform logits + + Assert.Equal(4, model.ForwardCalls.Count); + Assert.All(model.ForwardCalls, w => Assert.Equal(16, w.Length)); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(8, model.ForwardCalls[1][0]); + Assert.Equal(16, model.ForwardCalls[2][0]); + Assert.Equal(24, model.ForwardCalls[3][0]); + } + + [Fact] + public void SlidingWindow_EvaluatesEachWindowAsAnIndependentSequenceFromPositionZero() + { + // llama.cpp evaluates every chunk as a fresh sequence with positions restarting at 0. + // That is also what allows a corpus longer than the model's max sequence length to be + // scored at all -- absolute positions would run past it and throw. + var seenPositions = new List(); + + float[] Rows(int position, int vocab) + { + seenPositions.Add(position); + var row = new float[vocab]; + row[0] = 1f; + return row; + } + + var tokens = new int[40]; + using var model = new FakePerplexityModel(Vocab, maxContextLength: 16, returnsAllRows: true, Rows); + + PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 8)); + + // Never a position at or beyond the window length, however far into the corpus we are. + Assert.All(seenPositions, p => Assert.InRange(p, 0, 15)); + Assert.Contains(0, seenPositions); + } + + [Fact] + public void SlidingWindow_ScoresTargetsFromTheCorrectRow() + { + // Row i of a window predicts the window's token i+1. A row/target off-by-one shows up as + // a confident model scoring badly. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[(position + 1) % vocab] = 20f; // argmax == the id this row should predict + return row; + } + + // Window-relative: token at window offset j has id j % Vocab, so row j's target is + // (j+1) % Vocab -- exactly the argmax above. Stride == context keeps windows aligned to + // the same offsets, so this holds for every window. + var tokens = new int[64]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = (i % 16) % Vocab; + + using var model = new FakePerplexityModel(Vocab, 16, returnsAllRows: true, Rows); + var result = PerplexityEvaluator.Evaluate( + model, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 16, UnscoredPrefix: 8)); + + Assert.True(result.MeanNegativeLogLikelihood < 0.01, + $"expected confident predictions, got mean NLL {result.MeanNegativeLogLikelihood}"); + } + + [Fact] + public void SlidingWindow_RejectsUnscoredPrefixLeavingNothingToScore() + { + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + Assert.Throws(() => PerplexityEvaluator.Evaluate( + model, Tokens, + new PerplexityOptions(PerplexityMode.SlidingWindow, ContextLength: 16, Stride: 16, UnscoredPrefix: 16))); + } + + [Fact] + public void LlamaCppDefault_UsesNonOverlappingChunksScoringTheSecondHalf() + { + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + // L=16 => chunks at 0, 16 (32 would need tokens up to 48). Each scores 7, not 8: + // llama.cpp's count is n_ctx - n_ctx/2 - 1. + var result = PerplexityEvaluator.Evaluate( + model, tokens, PerplexityOptions.LlamaCppDefault(contextLength: 16)); + + Assert.Equal(2, result.WindowCount); + Assert.Equal(14, result.ScoredTokens); // 2 chunks x 7 scored + Assert.Equal(2, model.ForwardCalls.Count); + Assert.Equal(0, model.ForwardCalls[0][0]); + Assert.Equal(16, model.ForwardCalls[1][0]); // advances by the FULL window, not by 8 + } + + [Fact] + public void LlamaCppDefault_ScoresOneFewerTargetThanHalfTheWindow() + { + // llama.cpp: first = n_ctx/2, then `count += n_ctx - first - 1`, and the target for row j is + // token j+1 — so the token AT n_ctx/2 is context only and is never scored. Scoring it too + // would give n_ctx/2 targets per chunk instead of n_ctx/2 - 1: the same name, a different + // measurement, and a figure that is not comparable to a published one. + foreach (int context in new[] { 16, 64, 512 }) + { + var options = PerplexityOptions.LlamaCppDefault(context); + Assert.Equal(context / 2 + 1, options.UnscoredPrefix); + Assert.Equal(context / 2 - 1, context - options.UnscoredPrefix); + } + } + + [Fact] + public void StandardError_MatchesTheSampleVarianceOfPerTokenNll() + { + // Every target is token 0, and every row's only non-zero logit is at index 0 — so the + // target is always the argmax and its NLL depends solely on the row's peak height. That + // makes the whole set of per-token NLLs predictable in closed form, without reaching + // into the evaluator or replaying tensors. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[0] = position % 2 == 0 ? 2f : 0f; + return row; + } + + static double Nll(int position) + { + double peak = position % 2 == 0 ? 2.0 : 0.0; + return Math.Log(Math.Exp(peak) + (Vocab - 1)) - peak; + } + + var tokens = new int[40]; // all zero + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + + var result = PerplexityEvaluator.Evaluate(model, tokens, PerplexityOptions.LlamaCppDefault(16)); + + // L=16 => windows at 0 and 16; prefix 9 scores targets [s+9, s+16), i.e. rows 8..14. + var expectedNlls = new List(); + for (int window = 0; window < 2; window++) + for (int row = 8; row <= 14; row++) + expectedNlls.Add(Nll(row)); + + double mean = expectedNlls.Average(); + double variance = expectedNlls.Sum(v => (v - mean) * (v - mean)) / expectedNlls.Count; + double expected = Math.Sqrt(variance / (expectedNlls.Count - 1)) * Math.Exp(mean); + + Assert.Equal(expectedNlls.Count, result.ScoredTokens); + Assert.Equal(mean, result.MeanNegativeLogLikelihood, 9); + Assert.Equal(expected, result.StandardError, 9); + Assert.True(result.StandardError > 0); + } + + [Fact] + public void StandardError_IsZeroWhenEveryScoredTokenHasTheSameNll() + { + // A uniform distribution gives every target an identical NLL: zero variance, so the error + // bar must be exactly 0 rather than a NaN out of a negative rounding residue. + var tokens = Enumerable.Range(0, 40).ToArray(); + using var model = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, FakePerplexityModel.Uniform); + + var result = PerplexityEvaluator.Evaluate(model, tokens, PerplexityOptions.LlamaCppDefault(16)); + + Assert.Equal(0, result.StandardError); + } + + + [Fact] + public void LlamaCppDefault_AndContiguousTiling_ScoreDifferentTokenSets() + { + // The bug this guards: both schemes score the same COUNT, so a count check alone cannot + // distinguish them. Only a position-dependent signal reveals the different token sets. + // Row at position p predicts token p+1. Give the correct target a confidence that VARIES + // with p, so the mean NLL depends on which positions were scored, not merely how many. + static float[] Rows(int position, int vocab) + { + var row = new float[vocab]; + row[(position + 1) % vocab] = 1f + (position % 7); + return row; + } + + var tokens = new int[64]; + for (int i = 0; i < tokens.Length; i++) tokens[i] = i % Vocab; + + using var chunked = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + using var tiled = new FakePerplexityModel(Vocab, 64, returnsAllRows: true, Rows); + + var a = PerplexityEvaluator.Evaluate(chunked, tokens, PerplexityOptions.LlamaCppDefault(16)); + var b = PerplexityEvaluator.Evaluate( + tiled, tokens, new PerplexityOptions(PerplexityMode.SlidingWindow, 16, Stride: 8)); + + Assert.NotEqual(a.Perplexity, b.Perplexity, 6); + } +}