diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6264762 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,158 @@ +name: Release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + tag: + description: Existing v-prefixed tag to package + required: true + +permissions: + contents: write + id-token: write + attestations: write + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + archive: tar.gz + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + archive: tar.gz + - runner: macos-15-intel + target: x86_64-apple-darwin + archive: tar.gz + - runner: macos-14 + target: aarch64-apple-darwin + archive: tar.gz + - runner: windows-2022 + target: x86_64-pc-windows-msvc + archive: zip + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.88.0 + targets: ${{ matrix.target }} + - name: Verify release candidate + run: cargo test --workspace --locked + - name: Build binary + run: cargo build --release --locked --bin spectra --target ${{ matrix.target }} + - name: Package Unix binary + if: matrix.archive == 'tar.gz' + shell: bash + run: | + mkdir -p dist/package + cp target/${{ matrix.target }}/release/spectra dist/package/spectra + cp README.md LICENSE dist/package/ + tar -C dist/package -czf dist/spectra-${{ matrix.target }}.tar.gz . + - name: Package Windows binary + if: matrix.archive == 'zip' + shell: pwsh + run: | + New-Item -ItemType Directory -Force dist/package | Out-Null + Copy-Item target/${{ matrix.target }}/release/spectra.exe dist/package/ + Copy-Item README.md,LICENSE dist/package/ + Compress-Archive -Path dist/package/* -DestinationPath dist/spectra-${{ matrix.target }}.zip + - uses: actions/upload-artifact@v4 + with: + name: spectra-${{ matrix.target }} + path: dist/spectra-* + if-no-files-found: error + + release: + name: Sign and publish + needs: build + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: Generate checksums + working-directory: dist + run: sha256sum spectra-* > SHA256SUMS + - uses: sigstore/cosign-installer@v3 + - name: Sign artifacts with Sigstore + working-directory: dist + run: | + for artifact in spectra-* SHA256SUMS; do + cosign sign-blob --yes --bundle "${artifact}.sigstore.json" "$artifact" + done + - uses: actions/attest-build-provenance@v2 + with: + subject-path: dist/spectra-* + - name: Render package-manager manifests + shell: bash + env: + RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }} + run: packaging/render-manifests.sh "$RELEASE_TAG" dist/SHA256SUMS dist + - uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.event.inputs.tag || github.ref_name }} + generate_release_notes: true + files: | + dist/* + + publish-manifests: + name: Publish Homebrew and Scoop manifests + needs: release + runs-on: ubuntu-24.04 + env: + PACKAGE_REPOSITORIES_TOKEN: ${{ secrets.PACKAGE_REPOSITORIES_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: Regenerate manifests + env: + RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }} + run: | + (cd dist && sha256sum spectra-* > SHA256SUMS) + packaging/render-manifests.sh "$RELEASE_TAG" dist/SHA256SUMS dist + - name: Publish manifests + if: env.PACKAGE_REPOSITORIES_TOKEN != '' + env: + GH_TOKEN: ${{ secrets.PACKAGE_REPOSITORIES_TOKEN }} + RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }} + run: | + publish_manifest() { + repository=$1 + path=$2 + file=$3 + sha=$(gh api "repos/$repository/contents/$path" --jq .sha 2>/dev/null || true) + if [ -n "$sha" ]; then + gh api --method PUT "repos/$repository/contents/$path" \ + -f message="chore: update Spectra ${RELEASE_TAG}" \ + -f content="$(base64 -w0 "$file")" \ + -f sha="$sha" + else + gh api --method PUT "repos/$repository/contents/$path" \ + -f message="chore: update Spectra ${RELEASE_TAG}" \ + -f content="$(base64 -w0 "$file")" + fi + } + publish_manifest rankupgames/homebrew-tap Formula/spectra.rb dist/spectra.rb + publish_manifest rankupgames/scoop-bucket bucket/spectra.json dist/spectra.json diff --git a/Cargo.lock b/Cargo.lock index b923ddb..71788fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1223,7 +1223,7 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "spectra-context" -version = "0.3.0" +version = "0.4.0" dependencies = [ "atomic-write-file", "base64", @@ -1242,7 +1242,7 @@ dependencies = [ [[package]] name = "spectra-context-core" -version = "0.3.0" +version = "0.4.0" dependencies = [ "atomic-write-file", "ignore", @@ -1276,7 +1276,7 @@ dependencies = [ [[package]] name = "spectra-context-render" -version = "0.3.0" +version = "0.4.0" dependencies = [ "resvg", "spectra-context-core", diff --git a/Cargo.toml b/Cargo.toml index eab8264..62ca981 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.3.0" +version = "0.4.0" edition = "2024" license = "MIT" rust-version = "1.88" diff --git a/README.md b/README.md index 24b904d..1a2e0c5 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,26 @@ # Spectra -**A smaller, more useful memory for local AI coding agents.** +**An adaptive context runtime for local AI coding agents.** [![Rust 1.88+](https://img.shields.io/badge/Rust-1.88%2B-CE412B?logo=rust&logoColor=white)](https://www.rust-lang.org/) -[![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-38BDF8.svg)](https://github.com/rankupgames/Spectra/releases/tag/v0.3.0) +[![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-38BDF8.svg)](https://github.com/rankupgames/Spectra/releases/tag/v0.4.0) [![License: MIT](https://img.shields.io/badge/License-MIT-22C55E.svg)](LICENSE) [![Status: Prototype](https://img.shields.io/badge/Status-Prototype-F59E0B.svg)](#project-status) AI coding agents are good at working with code. They are less good at remembering a whole codebase without repeatedly loading file trees, source dumps, terminal logs, and old conversation into context. -Spectra is an experiment in fixing that. It gives an agent two things: a visual map of the codebase and a small, durable record of what has already happened. The goal is simple: spend fewer tokens rediscovering context, and more of them doing the actual work. +Spectra is an experiment in fixing that. Its adaptive context runtime selects the smallest useful packet for the next decision, remembers which evidence an exact agent session has already received, and creates a visual map only when one is requested. The goal is simple: spend fewer tokens rediscovering context, and more of them doing the actual work. ```text -Polyglot repository ──code adapters──▶ topology graph ──▶ PNG map + exact anchors -Agent lifecycle ──adapter hooks──▶ immutable ledger ──▶ bounded state context +Query + lifecycle ──▶ adaptive selector ──▶ budgeted evidence packet +Polyglot repository ──code adapters──▶ topology graph ──▶ exact anchors + optional PNG +Agent lifecycle ──adapter hooks──▶ immutable ledger ──▶ bounded continuity ``` Instead of dumping source up front, Spectra lets the model see the shape of the system, choose an exact `path:start-end` anchor, and read code once it knows what it is looking for. > [!IMPORTANT] -> Spectra is an early prototype. The v0.3 registry retains the complete CodeGraph v1.3.0 language and extension surface with 39 adapters. The harness-neutral Ledger is verified for Codex, Claude Code, and Gemini CLI; Cursor support is intentionally partial because it can reinject continuity only at session start. See [Project status](#project-status) before relying on it in production. +> Spectra is an early prototype. The v0.4 runtime retains the complete CodeGraph v1.3.0 language and extension surface with 39 adapters. The harness-neutral Ledger is verified for Codex, Claude Code, and Gemini CLI; Cursor support is intentionally partial because it can reinject continuity only at session start. See [Project status](#project-status) before relying on it in production. The [agent support contract](docs/agent-support.md) tracks topology and Ledger support separately so an MCP integration is never mistaken for lifecycle coverage. @@ -49,11 +50,8 @@ These numbers come from nine frozen prompts across pinned ripgrep, Tokio, and ru ## Quickstart -Spectra does not have prebuilt binaries yet, but Cargo can install the tagged v0.3.0 release directly from GitHub: - ### Requirements -- Rust 1.88 or newer - At least one supported local agent: Claude Code, Cursor, Codex, OpenCode, Hermes Agent, Gemini CLI, Antigravity, or Kiro - A repository containing at least one supported source language @@ -61,10 +59,29 @@ Any MCP client can also run `spectra serve --mcp` manually. ### 1. Install Spectra +macOS and Linux: + +```sh +curl --proto '=https' --tlsv1.2 -fsSL https://raw.githubusercontent.com/rankupgames/Spectra/v0.4.0/install.sh | sh +``` + +Windows PowerShell: + +```powershell +irm https://raw.githubusercontent.com/rankupgames/Spectra/v0.4.0/install.ps1 | iex +``` + +Package-manager and source-install alternatives: + ```sh -cargo install --git https://github.com/rankupgames/Spectra.git --tag v0.3.0 --bin spectra --locked +brew install rankupgames/tap/spectra +scoop bucket add rankupgames https://github.com/rankupgames/scoop-bucket +scoop install rankupgames/spectra +cargo install --git https://github.com/rankupgames/Spectra.git --tag v0.4.0 --bin spectra --locked ``` +Cargo installation requires Rust 1.88 or newer. Every release archive is covered by the published `SHA256SUMS`, Sigstore bundle, and build provenance; both direct installers verify the archive checksum before installing it. + ### 2. Connect your agents ```sh @@ -97,10 +114,16 @@ After that, there is nothing to babysit. The long-lived MCP process watches ever Ask your agent an architecture or navigation question, for example: ```text -Use Spectra to map how request routing reaches persistence. +Use Spectra to find how request routing reaches persistence. +``` + +Or inspect the adaptive packet directly: + +```sh +spectra context "how does request routing reach persistence" --path /path/to/project ``` -Or render a map directly: +Render a map only when the visual topology is useful: ```sh spectra map "how does request routing reach persistence" --path /path/to/project @@ -125,6 +148,8 @@ Normal use creates a project-local `.spectra/` directory containing generated st .spectra/ ├── index-v4.json incremental polyglot code index ├── ledger-v1.jsonl append-only context ledger +├── context-receipts-v1.json hashed per-session evidence receipts +├── metrics-v1.json local aggregate efficiency counters └── artifacts/ generated PNG and SVG maps ``` @@ -155,7 +180,11 @@ spectra sync [PATH] [--quiet] spectra autosync install [PATH] spectra autosync status [PATH] spectra autosync remove [PATH] +spectra context [--path PATH] [--token-budget 128..2000] [--intent auto|resume|locate|flow|change|inspect] + [--representation text|map] [--delivery delta|full] + [--source-harness HARNESS --session-id ID] [--cursor CURSOR] spectra map [--path PATH] [--max-nodes 1..96] [--out DIR] +spectra stats [--path PATH] [--json] [--reset] spectra serve --mcp spectra lifecycle ingest spectra hook [--agent codex|claude|gemini|cursor] @@ -171,20 +200,24 @@ The installer is idempotent and ownership-aware: it updates stale Spectra regist ## MCP interface -Spectra keeps the default MCP surface to two complementary tools: +Spectra advertises one tool by default: ```text -spectra_brief(query, projectPath?, tokenBudget?, detail?, source?) -spectra_map(query, projectPath?, maxNodes?) +spectra_context( + query, projectPath?, tokenBudget?, intent?, representation?, + delivery?, source?, cursor? +) ``` -Use `spectra_brief` as the first call when starting or resuming work. It combines bounded project-wide Ledger facts, synchronization health, ranked graph anchors, affected boundaries, and suggested next reads. Session state is included only when an exact `{harness, sessionId}` source is supplied. Compact mode defaults to 600 estimated tokens; `detail=source` substitutes bounded, line-numbered source windows and never creates an image or map artifact. +`spectra_context` routes `auto`, `resume`, `locate`, `flow`, `change`, and `inspect` intents through the existing Ledger and graph engines. It returns atomic continuity, anchor, relation, change, test, boundary, source-window, and next-action evidence as compact Context Packet v1 text. The default budget is 600 estimated tokens. Continuation cursors bind the query, intent, and index version, and fail as `cursor_stale` instead of mixing changed results. -Use `spectra_map` when a visual architecture view is useful. Its response contains an `image/png` content block followed by compact anchor metadata and never includes source bodies. Legacy snake-case parameter spellings remain accepted by every tool. +Text is the default. `representation=map` appends the existing PNG content block and identifies its cost as provider-controlled; the text packet remains budgeted independently. With an exact `{harness, sessionId}` source, `delivery=delta` suppresses evidence already delivered to that session. Without an exact source, Spectra safely returns a full packet and performs no deduplication. `delivery=full` resets the session baseline. -Change impact, typed paths, and the full CodeGraph-parity query pack are available without rebuilding. Set `SPECTRA_MCP_TOOLS=all`, or provide a comma-separated short-name allowlist such as `brief,map,changes,path,explore`. The available opt-in tools are: +All twelve v0.3 tools remain available without rebuilding. Set `SPECTRA_MCP_TOOLS=all`, or provide a comma-separated short-name allowlist such as `context,brief,map,changes,path,explore`. Existing allowlists and snake-case aliases remain valid. The legacy tools are: ```text +spectra_brief(query, projectPath?, tokenBudget?, detail?, source?) +spectra_map(query, projectPath?, maxNodes?) spectra_changes(projectPath?, base?, paths?, depth?, includeTests?, tokenBudget?) spectra_path(from, to, fromFile?, toFile?, mode?, maxHops?, projectPath?) spectra_explore(query, maxFiles?, projectPath?, tokenBudget?) @@ -242,6 +275,8 @@ Spectra should not become another transcript database. It deliberately keeps les - Source bodies are excluded from topology responses. - Prompts, assistant messages, patch bodies, and terminal output bodies are not written to the Ledger. +- Context receipts store only a salted session-key digest, evidence hashes, sequence metadata, and access metadata; corruption and write failure fail open to a full response. +- Efficiency metrics are local aggregate counters and are never networked. Set `SPECTRA_METRICS=off` to disable collection, inspect them with `spectra stats`, or explicitly clear them with `spectra stats --reset`. - Credential-shaped values are redacted before persistence. - Hook retries use correlation IDs so immutable events are not duplicated. - Index writers use an ownership-checked heartbeat lock across MCP, CLI, and Git-hook processes. @@ -255,9 +290,10 @@ Provider hooks remain fail-open and record only their documented lifecycle surfa ```sh cargo fmt --check -cargo clippy --workspace --all-targets -- -D warnings -cargo test --workspace -cargo build --release --workspace +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --locked +cargo build --release --workspace --locked +cargo run -p spectra-context --bin spectra-v04-gate -- benchmarks/results/reviewed-v0.4.json ``` The benchmark protocol, frozen prompts, raw evaluation data, and replay fixtures live under [`benchmarks/`](benchmarks/README.md). @@ -270,24 +306,28 @@ Implemented: - CodeGraph-parity server framework routes plus React/Next, SwiftUI, React Native, Expo Module, and Fabric client/native bridges - embedded JavaScript/TypeScript bridges, component rendering and event bindings, and conventional SvelteKit, Nuxt, and Astro page routes - query-focused deterministic PNG and SVG rendering -- bounded MCP image and anchor responses -- the complete CodeGraph v1.3.0 MCP query capability set, with a one-tool default and allowlist-enabled explore/search/traversal/node/files/status tools +- budgeted Context Packet v1 responses with deterministic intent routing, atomic evidence packing, bounded source windows, stale-safe continuation cursors, and explicit-only maps +- session-durable evidence deduplication with hashed receipts, bounded LRU storage, concurrent writers, corruption recovery, and fail-open delivery +- local privacy-safe efficiency metrics with opt-out, inspection, and explicit reset +- the complete CodeGraph v1.3.0 MCP query capability set, with `spectra_context` as the one-tool default and all twelve v0.3 tools behind compatible allowlists - automatic MCP installation for Claude Code, Cursor, Codex, OpenCode, Hermes Agent, Gemini CLI, Antigravity, and Kiro - automatic lifecycle-hook installation for Codex, Claude Code, Gemini CLI, and Cursor - append-only, per-session State Machine Ledger with replay, recovery, redaction, concurrency control, cross-harness project facts, and bounded projection - stable harness-neutral `spectra lifecycle ingest` JSON v1 protocol - deterministic, provider-backed, and recorded-hook regression suites - pinned real-repository parity gates covering framework routes and multimodal topology quality +- cross-platform release archives, checksums, Sigstore signing/provenance, checksum-verifying installers, and generated Homebrew/Scoop manifests +- a reviewed-report gate enforcing the v0.4 polyglot efficiency, solve-rate, repetition, call-count, budget, and privacy thresholds Not yet implemented: - per-prompt Cursor context reinjection (the host currently exposes reliable reinjection only at session start) - complete unified-shell interception -- packaged release binaries and automatic updater +- automatic updater - Tauri observability UI - public graph-extension SDK -The v0.3 release hardens that CodeGraph-parity topology around a harness-neutral continuity protocol, verified provider adapters, project-local setup, and a richer terminal workflow. Packaged installers that do not require a Rust toolchain remain future work. +The v0.4 release turns that topology and continuity foundation into an adaptive, text-first context runtime. Existing index-v4, ledger-v1, lifecycle-v1, MCP commands, hook installations, and legacy tool response contracts remain compatible. ## Contributing diff --git a/benchmarks/README.md b/benchmarks/README.md index 79a8ffc..053953e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -93,6 +93,37 @@ Codex, Claude, and Gemini must retain equivalent session facts. Cursor is gated The post-hook deterministic regression must preserve the 93.4% median reduction and 100% minimum fact-recall baselines recorded during prototype development. +## v0.4 adaptive-context release gate + +The v0.4 release uses [`v0.4-holdout.json`](v0.4-holdout.json): 20 public repositories pinned to exact commits across Rust, Python, JavaScript, TypeScript, Ruby, PHP, Java, C#, Go, Swift, Dart, and C++. Its five task templates—navigation, change impact, flow, repair, and resume—are instantiated once per repository for 100 tasks. Holdout repositories must not influence selector rules. Each task runs baseline and Spectra arms with the same harness, model, task instructions, provider settings, and clean repository state. + +Record provider input as separate schema, text, and image counts; record cached input as the provider-reported subset rather than adding it to the total. Also record output tokens, tool calls, latency, repeated context bytes, and final task success. The reviewed report format is defined by [`v0.4-report-schema.json`](v0.4-report-schema.json). A privacy review populates `forbidden_findings`; a release report must contain no source bodies, prompts, patches, terminal bodies, credentials, or raw session IDs in receipts or metrics. + +Validate a reviewed report with: + +```sh +cargo run --release -p spectra-context --bin spectra-v04-gate -- \ + benchmarks/results/reviewed-v0.4.json +``` + +The gate rejects fewer than 20 repositories, fewer than 100 unique tasks, fewer than two models or three harnesses, missing task categories, inconsistent provider-input accounting, or any privacy finding. It then requires at least 35% lower median provider input, at least 20% lower input at p75, solve rate within two percentage points of baseline, at least 70% fewer repeated context bytes, a median of at most two Spectra calls, and at least 95% budget-compliant text packets. Paid provider runs and task grading remain a reviewed release step; deterministic CI validates the gate itself and does not fabricate results. + +### Grok single-provider pilot + +The resumable Grok runner expands all five templates across the 20 pinned repositories. Both arms receive the same task, system instructions, model settings, and bounded `read_source` tool. The baseline arm receives literal `search_code`; the Spectra arm instead receives an initial 600-token `spectra_context` packet and may request another packet. Neither arm may edit files. Provider-reported input, cached input, output, latency, and cost are accumulated across the complete tool loop. Tool schema tokens are conservatively estimated because the xAI usage response reports total and cached input, not a schema/text split. + +```sh +benchmarks/materialize-v0.4.sh benchmarks/v0.4-holdout.json /path/to/v0.4-corpus +cargo build --release --locked --bin spectra --bin spectra-v04-grok-eval +target/release/spectra-v04-grok-eval \ + --corpus-root /path/to/v0.4-corpus \ + --output benchmarks/results/grok-v0.4-pilot +cargo run --release -p spectra-context --bin spectra-v04-gate -- \ + benchmarks/results/grok-v0.4-pilot/reviewed-v0.4.json --pilot +``` + +Raw responses and per-arm summaries are stored under the ignored result directory. Existing summaries are reused, so rerunning the same command safely resumes an interrupted paid evaluation. `--pilot` relaxes only the environment-count check; every corpus, task, efficiency, solve-rate, repetition, packet-budget, accounting, and privacy threshold remains unchanged. A Grok-only pass is evidence for iteration, not the final multi-environment release approval. + ## Agent efficiency scenarios [`fixtures/efficiency-tool-scenarios.json`](fixtures/efficiency-tool-scenarios.json) freezes three common agent workflows: resuming after failed verification, discovering worktree impact and tests, and tracing an A-to-B flow. The release test compares the previous focused-query call count with the composite `brief`, `changes`, or `path` call and requires at least 40% fewer median calls. Separate query tests gate token budgets, changed-path and verification retention, deterministic paths, session isolation, and exclusion of raw diffs, source bodies, terminal output, and credentials. diff --git a/benchmarks/materialize-v0.4.sh b/benchmarks/materialize-v0.4.sh new file mode 100755 index 0000000..173377f --- /dev/null +++ b/benchmarks/materialize-v0.4.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +manifest=${1:-benchmarks/v0.4-holdout.json} +corpus=${2:?corpus directory is required} + +mkdir -p "$corpus" +jq -r '.repositories[] | [.id, .url, .commit] | @tsv' "$manifest" | +while IFS="$(printf '\t')" read -r id url commit; do + destination="$corpus/$id" + if [ -d "$destination/.git" ]; then + actual=$(git -C "$destination" rev-parse HEAD) + if [ "$actual" != "$commit" ]; then + echo "spectra: $id is at $actual, expected $commit" >&2 + exit 1 + fi + echo "Reusing $id@$commit" + continue + fi + if [ -e "$destination" ]; then + echo "spectra: refusing to replace non-checkout $destination" >&2 + exit 1 + fi + echo "Fetching $id@$commit" + git init --quiet "$destination" + git -C "$destination" remote add origin "$url" + git -C "$destination" fetch --quiet --depth 1 origin "$commit" + git -C "$destination" checkout --quiet --detach FETCH_HEAD +done diff --git a/benchmarks/v0.4-grok-pilot-2026-07-20.md b/benchmarks/v0.4-grok-pilot-2026-07-20.md new file mode 100644 index 0000000..038dd1e --- /dev/null +++ b/benchmarks/v0.4-grok-pilot-2026-07-20.md @@ -0,0 +1,75 @@ +# Spectra v0.4 Grok single-provider pilot + +Date: 2026-07-20 +Model: `grok-4.5`, low reasoning effort +Label: single-provider pilot; not the final multi-environment release gate + +## Result + +The full frozen v0.4 holdout ran 100 paired tasks across 20 exact repository commits and all five task categories. The baseline arm used literal repository search plus bounded source reads. The Spectra arm received an initial 600-token `spectra_context` packet and could request more packets, plus the same bounded source reader. Neither arm could edit files. + +| Gate | Requirement | Grok pilot | Result | +| --- | ---: | ---: | --- | +| Median provider-input reduction | ≥35% | 60.4% | Pass | +| p75 provider-input reduction | ≥20% | 55.0% | Pass | +| Solve-rate regression | within −2 points | +20 points (57% → 77%) | Pass | +| Repeated-context reduction | ≥70% | 95.4% | Pass | +| Median Spectra calls | ≤2 | 5 | **Fail** | +| Text packets within budget | ≥95% | 88.7% | **Fail** | +| Receipt/metrics privacy scan | no findings | no findings | Pass | + +The explicit `spectra-v04-gate --pilot` check therefore fails, first reporting `median Spectra tool calls exceed two`. The packet-compliance threshold also fails. The pilot passes five of the seven performance/privacy thresholds, but it is not a release pass. + +## Aggregate usage + +| Metric | Baseline | Spectra | Difference | +| --- | ---: | ---: | ---: | +| Provider input tokens | 5,619,029 | 2,563,692 | −54.4% aggregate | +| Deterministically solved tasks | 57/100 | 77/100 | +20 tasks | +| Provider API calls | 660 | 600 | −9.1% | +| Retrieval calls | 1,850 | 1,011 | −45.4% | +| Repeated context bytes | 5,790 | 264 | −95.4% | +| Provider cost | $5.1560 | $2.9124 | −43.5% | + +Total provider cost for both arms was $8.0684. Grok made 532 Spectra context calls across 100 tasks, including each initial packet. The call distribution—not provider latency or repository parsing—is the main v0.4 adoption problem exposed by this run. + +## Results by task category + +| Category | Input reduction | Baseline solved | Spectra solved | Median Spectra calls | Packet compliance | +| --- | ---: | ---: | ---: | ---: | ---: | +| Navigation | 36.7% | 14/20 | 16/20 | 4 | 89.5% | +| Change impact | 52.1% | 16/20 | 17/20 | 6 | 88.8% | +| Flow | 33.0% | 11/20 | 12/20 | 5 | 96.3% | +| Repair | 66.9% | 0/20 | 13/20 | 6 | 85.0% | +| Resume | 67.9% | 16/20 | 19/20 | 5 | 84.3% | + +Repair tasks asked for diagnosis and the next focused verification rather than modifying the checkout. Their deterministic success rule requires a valid repository anchor plus an explicit test or verification recommendation. Many baseline repair arms exhausted the eight-turn ceiling before producing a final answer, so the 0/20 baseline repair result must not be read as an independently judged real-world repair rate. + +## Reproducibility and review boundaries + +- The corpus is [`v0.4-holdout.json`](v0.4-holdout.json), expanded deterministically to 100 tasks. +- The runner is `spectra-v04-grok-eval`; every API turn and arm summary is resumable. +- The ignored local result directory is `benchmarks/results/grok-v0.4-full-2026-07-20/`. +- It contains 200 arm summaries, all raw provider responses, 2,861 reconstructed local tool traces, 100 initial Context Packets, `grok-pilot-details.json`, and the gate-shaped `reviewed-v0.4.json`. +- Provider input, cached input, output, latency, and cost come from xAI response usage. The schema/text split is a conservative local estimate because xAI reports total and cached input rather than a separate tool-schema count. +- Success is a deterministic proxy requiring a substantive final answer and valid on-disk `path:line` anchors, with category-specific flow/test/next-action checks. It is not a blind human or independent-model judgment. +- Receipt and metrics scans found no raw task IDs, harness/session IDs, prompts, credentials, source bodies, or provider payloads. Raw responses and tool outputs remain only in the ignored benchmark result directory. +- This run covers one model, provider, and local harness. The normal gate still requires at least two models across three harnesses; `--pilot` relaxes only that environment-count check. + +## Recommended v0.4 follow-up + +Do not tune ranking rules to these holdout repositories. Address the generic runtime behaviors the pilot exposed: + +1. Make the first packet more explicitly sufficient and tell harnesses when another call is unlikely to add evidence. +2. Enforce a default per-task context-call ceiling of two, with continuation only when the packet reports material omitted evidence. +3. Fix packet overhead accounting so the final header, omission line, and continuation cursor remain inside the requested estimate. +4. Re-run this exact Grok corpus after those generic changes, then perform the required multi-model, multi-harness release evaluation. + +## Post-pilot corrections + +The two failed implementation checks were corrected after preserving the result above: + +- The Grok harness now permits one follow-up packet after the initial packet, removes `spectra_context` from subsequent tool schemas, and keeps bounded `read_source` available. +- Packet packing now reserves the estimated final header, omission line, and worst-case continuation cursor. The rendered `used≈` value converges on the estimate of the complete emitted packet. + +Targeted tests cover the two-call schema transition, atomic packing, separator accounting, and complete rendered packets at the public 128, 600, and 2,000-token boundaries. The paid 100-task corpus was intentionally not rerun; the original pilot measurements remain unchanged and continue to describe the pre-correction run. diff --git a/benchmarks/v0.4-holdout.json b/benchmarks/v0.4-holdout.json new file mode 100644 index 0000000..30fbace --- /dev/null +++ b/benchmarks/v0.4-holdout.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "frozen_at": "2026-07-20", + "repositories": [ + {"id":"ripgrep","url":"https://github.com/BurntSushi/ripgrep","commit":"59e318f5ace48db54f37bb67c152535bc17fa153","language":"Rust","public_boundary":"the rg CLI entry point","focus":"search execution and matching","verification":"cargo test --workspace"}, + {"id":"tokio","url":"https://github.com/tokio-rs/tokio","commit":"ac6869a431d9d7e2a81ce5309f00730741d3462a","language":"Rust","public_boundary":"tokio::spawn","focus":"task scheduling and execution","verification":"cargo test -p tokio --lib"}, + {"id":"django","url":"https://github.com/django/django","commit":"76e1bca1311ae7073a1fa4add6f9d19d709f0f09","language":"Python","public_boundary":"the WSGI/ASGI request handler","focus":"URL resolution and view dispatch","verification":"python -m tests.runtests urls"}, + {"id":"flask","url":"https://github.com/pallets/flask","commit":"36e4a824f340fdee7ed50937ba8e7f6bc7d17f81","language":"Python","public_boundary":"Flask.wsgi_app","focus":"request dispatch to a view","verification":"pytest tests"}, + {"id":"fastapi","url":"https://github.com/fastapi/fastapi","commit":"afe41126f624af30038cc8e17b2aaf60ebd4b838","language":"Python","public_boundary":"FastAPI route registration","focus":"request routing and dependency execution","verification":"pytest tests"}, + {"id":"express","url":"https://github.com/expressjs/express","commit":"ae6dd37680e3a00618d6c8a3e522f0ee4eeba1a4","language":"JavaScript","public_boundary":"express() request handling","focus":"router matching and middleware dispatch","verification":"npm test"}, + {"id":"nestjs","url":"https://github.com/nestjs/nest","commit":"41d9f37b320a4021da007fcbf2d9d578625f399c","language":"TypeScript","public_boundary":"controller route registration","focus":"HTTP routing to controller invocation","verification":"npm test"}, + {"id":"rails","url":"https://github.com/rails/rails","commit":"3ac4702080445968f7308f9877a474a446a462ab","language":"Ruby","public_boundary":"Rails route recognition","focus":"controller action dispatch","verification":"bin/test"}, + {"id":"laravel","url":"https://github.com/laravel/framework","commit":"ccd1bdee65b1c091d75986306401880d112f33b1","language":"PHP","public_boundary":"the HTTP kernel","focus":"router matching and controller dispatch","verification":"vendor/bin/phpunit"}, + {"id":"spring-petclinic","url":"https://github.com/spring-projects/spring-petclinic","commit":"f182358d02e4a68e52bdbabf55ca7800288511e7","language":"Java","public_boundary":"an owner HTTP endpoint","focus":"owner controller and repository persistence","verification":"./mvnw test"}, + {"id":"aspnetcore","url":"https://github.com/dotnet/aspnetcore","commit":"dfe3e58cc0f6e88d646bc2f78adf3220219a4165","language":"C#","public_boundary":"endpoint routing middleware","focus":"endpoint selection and request dispatch","verification":"./eng/build.sh -test"}, + {"id":"gin","url":"https://github.com/gin-gonic/gin","commit":"34dac209ffb6ef85cc78c5d217bbb7ad001d68fd","language":"Go","public_boundary":"Engine.ServeHTTP","focus":"route lookup and handler-chain execution","verification":"go test ./..."}, + {"id":"chi","url":"https://github.com/go-chi/chi","commit":"8b258c7bb28f97a5f2a856ff7ef962578fec9215","language":"Go","public_boundary":"Mux.ServeHTTP","focus":"route matching and handler dispatch","verification":"go test ./..."}, + {"id":"vapor","url":"https://github.com/vapor/vapor","commit":"94441c84e8111ef372ff2d6bea750b662cd6b827","language":"Swift","public_boundary":"RoutesBuilder route registration","focus":"request routing to a responder","verification":"swift test"}, + {"id":"shelf","url":"https://github.com/dart-lang/shelf","commit":"e9c742d10432bae93d8a72a8840c2e701a6e2134","language":"Dart","public_boundary":"a shelf Handler","focus":"middleware pipeline and response production","verification":"dart test"}, + {"id":"vue-core","url":"https://github.com/vuejs/core","commit":"fa2885d8c48768d26f1666a01bd540ffe3b20f9b","language":"TypeScript","public_boundary":"createApp","focus":"component mounting and update scheduling","verification":"pnpm test"}, + {"id":"svelte-kit","url":"https://github.com/sveltejs/kit","commit":"adc4c5be542ecf7da93ee8d0e90e9d13b952998b","language":"TypeScript","public_boundary":"the server request responder","focus":"route resolution and endpoint rendering","verification":"pnpm test"}, + {"id":"astro","url":"https://github.com/withastro/astro","commit":"cf9ffc72258b23211eb07f4f38176756812b373d","language":"TypeScript","public_boundary":"an incoming server request","focus":"route matching and page rendering","verification":"pnpm test"}, + {"id":"terraform","url":"https://github.com/hashicorp/terraform","commit":"5c8717d62e80b8382cbed292919171d0ffb58237","language":"Go","public_boundary":"the terraform plan command","focus":"plan graph construction and walking","verification":"go test ./..."}, + {"id":"fmt","url":"https://github.com/fmtlib/fmt","commit":"7b4ef1c8145f21e126167f52a594909dde8c89fe","language":"C++","public_boundary":"fmt::format","focus":"format parsing and argument rendering","verification":"cmake --build build --target test"} + ], + "task_templates": [ + {"id":"navigation","category":"navigation","prompt":"Locate the primary public boundary for {focus}, starting from {public_boundary}. Return the smallest set of exact source anchors needed to continue implementation work."}, + {"id":"change-impact","category":"change-impact","prompt":"Assume the implementation behind {focus} changes without a public signature change. Identify the affected callers, public boundaries, and most relevant tests. Do not edit files."}, + {"id":"flow","category":"flow","prompt":"Trace the deterministic runtime flow from {public_boundary} to {focus}. Return ordered exact anchors and label any uncertain cross-boundary hop."}, + {"id":"repair","category":"repair","prompt":"The check `{verification}` reports a behavioral failure owned by {focus}. Locate the smallest likely repair boundary and the focused verification to run next. Do not edit files."}, + {"id":"resume","category":"resume","prompt":"An earlier turn changed code around {focus} and stopped before `{verification}`. Resume by recovering the changed boundary, current verification state, and next action without repeating already delivered evidence."} + ] +} diff --git a/benchmarks/v0.4-report-schema.json b/benchmarks/v0.4-report-schema.json new file mode 100644 index 0000000..0fc5d09 --- /dev/null +++ b/benchmarks/v0.4-report-schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/rankupgames/Spectra/benchmarks/v0.4-report-schema.json", + "title": "Spectra v0.4 reviewed efficiency report", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "environments", "tasks", "forbidden_findings"], + "properties": { + "schema_version": { "const": 1 }, + "environments": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "harness", "model"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "harness": { "type": "string", "minLength": 1 }, + "model": { "type": "string", "minLength": 1 } + } + } + }, + "tasks": { + "type": "array", + "minItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "repository", "environment_id", "category", "baseline", "spectra", "packets_within_budget", "packets_total"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "repository": { "type": "string", "minLength": 1 }, + "environment_id": { "type": "string", "minLength": 1 }, + "category": { "enum": ["navigation", "change-impact", "flow", "repair", "resume"] }, + "baseline": { "$ref": "#/$defs/arm" }, + "spectra": { "$ref": "#/$defs/arm" }, + "packets_within_budget": { "type": "integer", "minimum": 0 }, + "packets_total": { "type": "integer", "minimum": 1 } + } + } + }, + "forbidden_findings": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + }, + "$defs": { + "arm": { + "type": "object", + "additionalProperties": false, + "required": ["input_tokens", "input_schema_tokens", "input_text_tokens", "image_tokens", "cached_input_tokens", "output_tokens", "solved", "tool_calls", "latency_ms", "repeated_context_bytes"], + "properties": { + "input_tokens": { "type": "integer", "minimum": 1 }, + "input_schema_tokens": { "type": "integer", "minimum": 0 }, + "input_text_tokens": { "type": "integer", "minimum": 1 }, + "image_tokens": { "type": "integer", "minimum": 0 }, + "cached_input_tokens": { "type": "integer", "minimum": 0 }, + "output_tokens": { "type": "integer", "minimum": 1 }, + "solved": { "type": "boolean" }, + "tool_calls": { "type": "integer", "minimum": 0 }, + "latency_ms": { "type": "integer", "minimum": 1 }, + "repeated_context_bytes": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/crates/spectra-cli/Cargo.toml b/crates/spectra-cli/Cargo.toml index e95275d..f8c640b 100644 --- a/crates/spectra-cli/Cargo.toml +++ b/crates/spectra-cli/Cargo.toml @@ -27,8 +27,8 @@ rmcp.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true -spectra-core = { package = "spectra-context-core", version = "0.3.0", path = "../spectra-core" } -spectra-render = { package = "spectra-context-render", version = "0.3.0", path = "../spectra-render" } +spectra-core = { package = "spectra-context-core", version = "0.4.0", path = "../spectra-core" } +spectra-render = { package = "spectra-context-render", version = "0.4.0", path = "../spectra-render" } tokio.workspace = true [[bin]] @@ -50,3 +50,11 @@ path = "src/bin/spectra-ledger-grok-eval.rs" [[bin]] name = "spectra-adapter-eval" path = "src/bin/spectra-adapter-eval.rs" + +[[bin]] +name = "spectra-v04-gate" +path = "src/bin/spectra-v04-gate.rs" + +[[bin]] +name = "spectra-v04-grok-eval" +path = "src/bin/spectra-v04-grok-eval.rs" diff --git a/crates/spectra-cli/src/autosync.rs b/crates/spectra-cli/src/autosync.rs index e8ba727..a4be756 100644 --- a/crates/spectra-cli/src/autosync.rs +++ b/crates/spectra-cli/src/autosync.rs @@ -538,7 +538,11 @@ mod tests { let autosync = AutoSync::with_debounce(Duration::from_millis(100)); let initial = autosync.ensure_project(&root); assert!(initial.active, "{:?}", initial.last_error); - assert_eq!(initial.sync_count, 1); + wait_for(|| { + let status = autosync.status(&root); + status.sync_count >= 1 && status.pending == 0 + }); + let initial = autosync.status(&root); fs::write(&source, "pub fn second() {}\n").unwrap(); wait_for(|| { @@ -626,7 +630,9 @@ mod tests { } fn wait_for(condition: impl Fn() -> bool) { - let deadline = Instant::now() + Duration::from_secs(5); + // FSEvents delivery can lag behind the polling fallback on busy hosted + // macOS runners, especially while the workspace is still compiling. + let deadline = Instant::now() + Duration::from_secs(15); while !condition() { assert!( Instant::now() < deadline, diff --git a/crates/spectra-cli/src/bin/spectra-v04-gate.rs b/crates/spectra-cli/src/bin/spectra-v04-gate.rs new file mode 100644 index 0000000..6231410 --- /dev/null +++ b/crates/spectra-cli/src/bin/spectra-v04-gate.rs @@ -0,0 +1,429 @@ +use std::{collections::BTreeSet, fs, path::PathBuf}; + +use clap::Parser; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Parser)] +#[command(about = "Verify a reviewed Spectra v0.4 end-to-end evaluation report")] +struct Args { + report: PathBuf, + #[arg(long)] + json: bool, + /// Accept a single model/harness for an explicitly labeled pilot report. + #[arg(long)] + pilot: bool, +} + +#[derive(Clone, Debug, Deserialize)] +struct Report { + schema_version: u32, + environments: Vec, + tasks: Vec, + #[serde(default)] + forbidden_findings: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +struct Environment { + id: String, + harness: String, + model: String, +} + +#[derive(Clone, Debug, Deserialize)] +struct Task { + id: String, + repository: String, + environment_id: String, + category: String, + baseline: Arm, + spectra: Arm, + packets_within_budget: u64, + packets_total: u64, +} + +#[derive(Clone, Debug, Deserialize)] +struct Arm { + input_tokens: u64, + input_schema_tokens: u64, + input_text_tokens: u64, + image_tokens: u64, + cached_input_tokens: u64, + output_tokens: u64, + solved: bool, + tool_calls: u64, + latency_ms: u64, + repeated_context_bytes: u64, +} + +#[derive(Clone, Debug, Serialize)] +struct Summary { + repositories: usize, + tasks: usize, + models: usize, + harnesses: usize, + median_input_reduction: f64, + p75_input_reduction: f64, + baseline_solve_rate: f64, + spectra_solve_rate: f64, + repeated_context_reduction: f64, + median_spectra_tool_calls: f64, + packets_within_budget: f64, +} + +fn main() -> Result<(), Box> { + let args = Args::parse(); + let report: Report = serde_json::from_slice(&fs::read(args.report)?)?; + let summary = evaluate(&report, args.pilot)?; + if args.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else { + println!( + "v0.4 gate passed: repositories={} tasks={} median_input_reduction={:.1}% p75_input_reduction={:.1}% solve={:.1}% repeated_context_reduction={:.1}% median_tool_calls={:.1} budget_compliance={:.1}%", + summary.repositories, + summary.tasks, + summary.median_input_reduction * 100.0, + summary.p75_input_reduction * 100.0, + summary.spectra_solve_rate * 100.0, + summary.repeated_context_reduction * 100.0, + summary.median_spectra_tool_calls, + summary.packets_within_budget * 100.0, + ); + } + Ok(()) +} + +fn evaluate(report: &Report, pilot: bool) -> Result> { + if report.schema_version != 1 { + return Err(format!("unsupported report schema {}", report.schema_version).into()); + } + let repositories = report + .tasks + .iter() + .map(|task| task.repository.as_str()) + .collect::>(); + let models = report + .environments + .iter() + .map(|environment| environment.model.as_str()) + .collect::>(); + let harnesses = report + .environments + .iter() + .map(|environment| environment.harness.as_str()) + .collect::>(); + if repositories.len() < 20 || report.tasks.len() < 100 { + return Err("v0.4 requires at least 20 repositories and 100 tasks".into()); + } + if !pilot && (models.len() < 2 || harnesses.len() < 3) { + return Err("v0.4 requires at least two models across three harnesses".into()); + } + let environment_ids = report + .environments + .iter() + .map(|environment| environment.id.as_str()) + .collect::>(); + if environment_ids.len() != report.environments.len() + || report + .tasks + .iter() + .any(|task| !environment_ids.contains(task.environment_id.as_str())) + { + return Err("environment IDs must be unique and every task must reference one".into()); + } + let categories = report + .tasks + .iter() + .map(|task| task.category.as_str()) + .collect::>(); + for required in ["navigation", "change-impact", "flow", "repair", "resume"] { + if !categories.contains(required) { + return Err(format!("v0.4 report is missing {required} tasks").into()); + } + } + if !report.forbidden_findings.is_empty() { + return Err(format!( + "privacy scan reported: {}", + report.forbidden_findings.join(", ") + ) + .into()); + } + let unique_ids = report + .tasks + .iter() + .map(|task| task.id.as_str()) + .collect::>(); + if unique_ids.len() != report.tasks.len() { + return Err("task IDs must be unique".into()); + } + if report.tasks.iter().any(|task| { + task.baseline.input_tokens == 0 + || task.spectra.input_tokens == 0 + || task.baseline.input_text_tokens == 0 + || task.spectra.input_text_tokens == 0 + || task.baseline.output_tokens == 0 + || task.spectra.output_tokens == 0 + || task.baseline.latency_ms == 0 + || task.spectra.latency_ms == 0 + || task.packets_total == 0 + }) { + return Err( + "text/input/output token counts, latency, and packet totals must be nonzero".into(), + ); + } + for task in &report.tasks { + for arm in [&task.baseline, &task.spectra] { + let counted_input = arm.input_schema_tokens + arm.input_text_tokens + arm.image_tokens; + if counted_input != arm.input_tokens || arm.cached_input_tokens > arm.input_tokens { + return Err( + format!("{} has inconsistent provider input accounting", task.id).into(), + ); + } + } + } + + let baseline_tokens = report + .tasks + .iter() + .map(|task| task.baseline.input_tokens as f64) + .collect::>(); + let spectra_tokens = report + .tasks + .iter() + .map(|task| task.spectra.input_tokens as f64) + .collect::>(); + let median_input_reduction = reduction( + percentile(&baseline_tokens, 0.5), + percentile(&spectra_tokens, 0.5), + ); + let p75_input_reduction = reduction( + percentile(&baseline_tokens, 0.75), + percentile(&spectra_tokens, 0.75), + ); + let baseline_solve_rate = rate( + report + .tasks + .iter() + .filter(|task| task.baseline.solved) + .count(), + report.tasks.len(), + ); + let spectra_solve_rate = rate( + report + .tasks + .iter() + .filter(|task| task.spectra.solved) + .count(), + report.tasks.len(), + ); + let baseline_repeated = report + .tasks + .iter() + .map(|task| task.baseline.repeated_context_bytes) + .sum::(); + let spectra_repeated = report + .tasks + .iter() + .map(|task| task.spectra.repeated_context_bytes) + .sum::(); + let repeated_context_reduction = reduction(baseline_repeated as f64, spectra_repeated as f64); + let median_spectra_tool_calls = percentile( + &report + .tasks + .iter() + .map(|task| task.spectra.tool_calls as f64) + .collect::>(), + 0.5, + ); + let packets_within = report + .tasks + .iter() + .map(|task| task.packets_within_budget) + .sum::(); + let packets_total = report + .tasks + .iter() + .map(|task| task.packets_total) + .sum::(); + let packets_within_budget = rate(packets_within as usize, packets_total as usize); + + let summary = Summary { + repositories: repositories.len(), + tasks: report.tasks.len(), + models: models.len(), + harnesses: harnesses.len(), + median_input_reduction, + p75_input_reduction, + baseline_solve_rate, + spectra_solve_rate, + repeated_context_reduction, + median_spectra_tool_calls, + packets_within_budget, + }; + if median_input_reduction < 0.35 { + return Err("median provider-input reduction is below 35%".into()); + } + if p75_input_reduction < 0.20 { + return Err("p75 provider-input reduction is below 20%".into()); + } + if baseline_solve_rate - spectra_solve_rate > 0.02 + f64::EPSILON { + return Err("Spectra solve rate is more than two points below baseline".into()); + } + if repeated_context_reduction < 0.70 { + return Err("repeated-context reduction is below 70%".into()); + } + if median_spectra_tool_calls > 2.0 { + return Err("median Spectra tool calls exceed two".into()); + } + if packets_within_budget < 0.95 { + return Err("fewer than 95% of packets stayed within budget".into()); + } + Ok(summary) +} + +fn percentile(values: &[f64], percentile: f64) -> f64 { + let mut values = values.to_vec(); + values.sort_by(f64::total_cmp); + let index = ((values.len() as f64 * percentile).ceil() as usize) + .saturating_sub(1) + .min(values.len().saturating_sub(1)); + values[index] +} + +fn reduction(baseline: f64, spectra: f64) -> f64 { + if baseline == 0.0 { + 0.0 + } else { + 1.0 - spectra / baseline + } +} + +fn rate(part: usize, total: usize) -> f64 { + part as f64 / total as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Value; + + #[test] + fn frozen_holdout_expands_to_twenty_repositories_and_one_hundred_tasks() { + let manifest: Value = + serde_json::from_str(include_str!("../../../../benchmarks/v0.4-holdout.json")).unwrap(); + let repositories = manifest["repositories"].as_array().unwrap(); + let templates = manifest["task_templates"].as_array().unwrap(); + assert_eq!(repositories.len(), 20); + assert_eq!(templates.len(), 5); + assert_eq!(repositories.len() * templates.len(), 100); + assert_eq!( + repositories + .iter() + .filter_map(|repository| repository["commit"].as_str()) + .filter( + |commit| commit.len() == 40 && commit.chars().all(|ch| ch.is_ascii_hexdigit()) + ) + .count(), + 20 + ); + } + + fn passing_report() -> Report { + Report { + schema_version: 1, + environments: vec![ + Environment { + id: "codex-frontier".into(), + harness: "codex".into(), + model: "frontier".into(), + }, + Environment { + id: "claude-frontier".into(), + harness: "claude".into(), + model: "frontier".into(), + }, + Environment { + id: "gemini-small".into(), + harness: "gemini".into(), + model: "small".into(), + }, + ], + tasks: (0..100) + .map(|index| Task { + id: format!("task-{index}"), + repository: format!("repo-{}", index % 20), + environment_id: ["codex-frontier", "claude-frontier", "gemini-small"] + [index % 3] + .into(), + category: ["navigation", "change-impact", "flow", "repair", "resume"] + [index % 5] + .into(), + baseline: Arm { + input_tokens: 1_000, + input_schema_tokens: 100, + input_text_tokens: 900, + image_tokens: 0, + cached_input_tokens: 200, + output_tokens: 100, + solved: true, + tool_calls: 5, + latency_ms: 1_000, + repeated_context_bytes: 1_000, + }, + spectra: Arm { + input_tokens: 500, + input_schema_tokens: 50, + input_text_tokens: 450, + image_tokens: 0, + cached_input_tokens: 100, + output_tokens: 100, + solved: true, + tool_calls: 2, + latency_ms: 500, + repeated_context_bytes: 200, + }, + packets_within_budget: 1, + packets_total: 1, + }) + .collect(), + forbidden_findings: Vec::new(), + } + } + + #[test] + fn accepts_a_report_that_meets_every_release_gate() { + let summary = evaluate(&passing_report(), false).unwrap(); + assert_eq!(summary.repositories, 20); + assert_eq!(summary.tasks, 100); + assert_eq!(summary.median_input_reduction, 0.5); + } + + #[test] + fn rejects_privacy_findings_and_regressions() { + let mut report = passing_report(); + report.forbidden_findings.push("raw session id".into()); + assert!( + evaluate(&report, false) + .unwrap_err() + .to_string() + .contains("privacy") + ); + report.forbidden_findings.clear(); + report.tasks[0].spectra.input_tokens = 2_000; + for task in &mut report.tasks[1..] { + task.spectra.input_tokens = 900; + } + assert!(evaluate(&report, false).is_err()); + } + + #[test] + fn pilot_mode_allows_one_grok_environment_without_weakening_release_mode() { + let mut report = passing_report(); + report.environments.truncate(1); + for task in &mut report.tasks { + task.environment_id = "codex-frontier".into(); + } + assert!(evaluate(&report, false).is_err()); + assert!(evaluate(&report, true).is_ok()); + } +} diff --git a/crates/spectra-cli/src/bin/spectra-v04-grok-eval.rs b/crates/spectra-cli/src/bin/spectra-v04-grok-eval.rs new file mode 100644 index 0000000..c2f40dc --- /dev/null +++ b/crates/spectra-cli/src/bin/spectra-v04-grok-eval.rs @@ -0,0 +1,1053 @@ +use std::{ + collections::BTreeSet, + fs, + io::Write, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use clap::{Parser, ValueEnum}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use spectra_core::estimate_tokens; + +const SYSTEM_PROMPT: &str = "You are evaluating a local code repository. Use only the supplied local tools. Do not edit files. Return a concise answer with exact path:start-end anchors for every material claim. For flows, order the anchors. For change or repair work, name focused tests. Stop once the evidence is sufficient."; +const MAX_TOOL_OUTPUT: usize = 30_000; +const MAX_SPECTRA_CONTEXT_CALLS: u64 = 2; + +#[derive(Debug, Parser)] +#[command(about = "Run the complete Spectra v0.4 holdout with Grok")] +struct Args { + #[arg(long, default_value = "benchmarks/v0.4-holdout.json")] + manifest: PathBuf, + #[arg(long)] + corpus_root: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long, default_value = "target/release/spectra")] + spectra_bin: PathBuf, + #[arg(long, default_value = ".env")] + env_file: PathBuf, + #[arg(long, default_value = "grok-4.5")] + model: String, + #[arg(long, default_value_t = 600, value_parser = clap::value_parser!(u16).range(128..=2000))] + token_budget: u16, + #[arg(long, default_value_t = 8, value_parser = clap::value_parser!(u8).range(1..=16))] + max_turns: u8, + /// Run at most this many expanded tasks; zero runs all 100. + #[arg(long, default_value_t = 0)] + limit: usize, + #[arg(long)] + repository: Vec, + #[arg(long)] + category: Vec, + /// Re-execute cached function calls locally to reconstruct tool traces. + /// Makes no provider requests when arm summaries already exist. + #[arg(long)] + replay_tool_traces: bool, +} + +#[derive(Clone, Debug, Deserialize)] +struct Manifest { + schema_version: u32, + repositories: Vec, + task_templates: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +struct Repository { + id: String, + commit: String, + public_boundary: String, + focus: String, + verification: String, +} + +#[derive(Clone, Debug, Deserialize)] +struct TaskTemplate { + id: String, + category: String, + prompt: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum ArmKind { + Baseline, + Spectra, +} + +impl ArmKind { + fn as_str(self) -> &'static str { + match self { + Self::Baseline => "baseline", + Self::Spectra => "spectra", + } + } +} + +#[derive(Clone, Debug)] +struct ExpandedTask { + id: String, + repository: String, + category: String, + prompt: String, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ArmResult { + input_tokens: u64, + cached_input_tokens: u64, + output_tokens: u64, + latency_ms: u64, + api_calls: u64, + retrieval_calls: u64, + spectra_calls: u64, + repeated_context_bytes: u64, + packets_within_budget: u64, + packets_total: u64, + cost_usd: f64, + valid_anchors: usize, + solved: bool, + answer: String, +} + +#[derive(Clone, Debug, Serialize)] +struct GateReport { + schema_version: u32, + environments: Vec, + tasks: Vec, + forbidden_findings: Vec, +} + +#[derive(Clone, Debug, Serialize)] +struct Environment { + id: String, + harness: String, + model: String, +} + +#[derive(Clone, Debug, Serialize)] +struct GateTask { + id: String, + repository: String, + environment_id: String, + category: String, + baseline: GateArm, + spectra: GateArm, + packets_within_budget: u64, + packets_total: u64, +} + +#[derive(Clone, Debug, Serialize)] +struct GateArm { + input_tokens: u64, + input_schema_tokens: u64, + input_text_tokens: u64, + image_tokens: u64, + cached_input_tokens: u64, + output_tokens: u64, + solved: bool, + tool_calls: u64, + latency_ms: u64, + repeated_context_bytes: u64, +} + +#[derive(Clone, Debug, Serialize)] +struct DetailReport { + schema_version: u32, + label: &'static str, + generated_at_unix_ms: u128, + model: String, + input_schema_accounting: &'static str, + tasks: Vec, +} + +#[derive(Clone, Debug, Serialize)] +struct DetailTask { + id: String, + repository: String, + category: String, + baseline: ArmResult, + spectra: ArmResult, +} + +#[derive(Default)] +struct DeliveryAccounting { + seen: BTreeSet, + repeated_bytes: u64, + packets_within_budget: u64, + packets_total: u64, +} + +fn main() -> Result<(), Box> { + let args = Args::parse(); + let manifest: Manifest = serde_json::from_slice(&fs::read(&args.manifest)?)?; + if manifest.schema_version != 1 { + return Err(format!("unsupported holdout schema {}", manifest.schema_version).into()); + } + let key = api_key(&args.env_file)?; + let tasks = expand_tasks(&manifest, &args); + if tasks.is_empty() { + return Err("no holdout tasks matched the filters".into()); + } + fs::create_dir_all(&args.output)?; + let mut details = Vec::new(); + for (index, task) in tasks.iter().enumerate() { + let repository = args.corpus_root.join(&task.repository); + verify_checkout(&repository, repository_commit(&manifest, &task.repository)?)?; + eprintln!( + "[{}/{}] {}/{} baseline", + index + 1, + tasks.len(), + task.repository, + task.category + ); + let baseline = run_arm(&args, &key, task, &repository, ArmKind::Baseline)?; + eprintln!( + "[{}/{}] {}/{} spectra", + index + 1, + tasks.len(), + task.repository, + task.category + ); + let spectra = run_arm(&args, &key, task, &repository, ArmKind::Spectra)?; + details.push(DetailTask { + id: task.id.clone(), + repository: task.repository.clone(), + category: task.category.clone(), + baseline, + spectra, + }); + write_reports(&args, &manifest, &details)?; + } + let (input_reduction, cost) = summary(&details); + println!( + "Completed {} paired tasks with {}: aggregate input reduction={:.1}% cost=${cost:.4}", + details.len(), + args.model, + input_reduction * 100.0 + ); + println!( + "Wrote {}", + args.output.join("grok-pilot-details.json").display() + ); + println!("Wrote {}", args.output.join("reviewed-v0.4.json").display()); + Ok(()) +} + +fn expand_tasks(manifest: &Manifest, args: &Args) -> Vec { + let mut tasks = Vec::new(); + for repository in &manifest.repositories { + if !args.repository.is_empty() && !args.repository.contains(&repository.id) { + continue; + } + for template in &manifest.task_templates { + if !args.category.is_empty() && !args.category.contains(&template.category) { + continue; + } + let prompt = template + .prompt + .replace("{focus}", &repository.focus) + .replace("{public_boundary}", &repository.public_boundary) + .replace("{verification}", &repository.verification); + tasks.push(ExpandedTask { + id: format!("{}-{}", repository.id, template.id), + repository: repository.id.clone(), + category: template.category.clone(), + prompt, + }); + if args.limit != 0 && tasks.len() >= args.limit { + return tasks; + } + } + } + tasks +} + +fn run_arm( + args: &Args, + key: &str, + task: &ExpandedTask, + repository: &Path, + arm: ArmKind, +) -> Result> { + let arm_dir = args + .output + .join("artifacts") + .join(&task.repository) + .join(&task.category) + .join(arm.as_str()); + fs::create_dir_all(&arm_dir)?; + let summary_path = arm_dir.join("summary.json"); + if summary_path.exists() { + if args.replay_tool_traces { + replay_tool_traces(args, task, repository, arm, &arm_dir)?; + } + return Ok(serde_json::from_slice(&fs::read(summary_path)?)?); + } + let mut delivery = DeliveryAccounting::default(); + let mut result = ArmResult::default(); + let initial_context = if arm == ArmKind::Spectra { + result.spectra_calls += 1; + let output = spectra_context(args, repository, task, &task.prompt, &task.category, "full")?; + account_output(&mut delivery, &output); + account_packet(&mut delivery, &output); + fs::write(arm_dir.join("initial-context.txt"), &output)?; + Some(output) + } else { + None + }; + let mut history = vec![ + json!({"role":"system", "content":SYSTEM_PROMPT}), + json!({ + "role":"user", + "content": match initial_context { + Some(context) => format!("Task: {}\n\nInitial Spectra Context Packet:\n{}", task.prompt, context), + None => format!("Task: {}", task.prompt), + } + }), + ]; + for turn in 0..args.max_turns { + let tools = tool_schemas( + arm, + arm != ArmKind::Spectra || result.spectra_calls < MAX_SPECTRA_CONTEXT_CALLS, + ); + let raw_path = arm_dir.join(format!("turn-{turn}.json")); + let (response, elapsed) = api_response(key, &args.model, &history, &tools, &raw_path)?; + result.latency_ms = result.latency_ms.saturating_add(elapsed as u64); + result.api_calls += 1; + add_usage(&mut result, &response); + let outputs = response["output"].as_array().cloned().unwrap_or_default(); + history.extend(outputs.iter().cloned()); + let calls = outputs + .iter() + .filter(|item| item["type"] == "function_call") + .cloned() + .collect::>(); + if calls.is_empty() { + result.answer = extract_answer(&response); + break; + } + for (call_index, call) in calls.into_iter().enumerate() { + let name = call["name"].as_str().unwrap_or_default(); + let arguments: Value = serde_json::from_str(call["arguments"].as_str().unwrap_or("{}")) + .unwrap_or_else(|_| json!({})); + let context_limit_reached = + name == "spectra_context" && result.spectra_calls >= MAX_SPECTRA_CONTEXT_CALLS; + let output = if context_limit_reached { + "context_limit reached; use the supplied anchors with read_source and answer." + .into() + } else { + execute_tool(args, repository, task, arm, name, &arguments)? + }; + result.retrieval_calls += 1; + if name == "spectra_context" && !context_limit_reached { + result.spectra_calls += 1; + account_packet(&mut delivery, &output); + } + account_output(&mut delivery, &output); + write_json( + &arm_dir.join(format!("tool-{turn}-{call_index}.json")), + &json!({"name":name,"arguments":arguments,"output":&output}), + )?; + history.push(json!({ + "type":"function_call_output", + "call_id":call["call_id"].as_str().unwrap_or_default(), + "output":output, + })); + } + } + result.repeated_context_bytes = delivery.repeated_bytes; + result.packets_within_budget = delivery.packets_within_budget; + result.packets_total = delivery.packets_total; + result.valid_anchors = valid_anchor_count(repository, &result.answer); + result.solved = deterministic_success(task, &result); + write_json(&summary_path, &result)?; + Ok(result) +} + +fn replay_tool_traces( + args: &Args, + task: &ExpandedTask, + repository: &Path, + arm: ArmKind, + arm_dir: &Path, +) -> Result<(), Box> { + if arm == ArmKind::Spectra { + let initial = + spectra_context(args, repository, task, &task.prompt, &task.category, "full")?; + fs::write(arm_dir.join("initial-context.txt"), initial)?; + } + for turn in 0..args.max_turns { + let raw_path = arm_dir.join(format!("turn-{turn}.json")); + if !raw_path.exists() { + break; + } + let envelope: Value = serde_json::from_slice(&fs::read(raw_path)?)?; + let calls = envelope["response"]["output"] + .as_array() + .into_iter() + .flatten() + .filter(|item| item["type"] == "function_call"); + for (call_index, call) in calls.enumerate() { + let name = call["name"].as_str().unwrap_or_default(); + let arguments: Value = serde_json::from_str(call["arguments"].as_str().unwrap_or("{}")) + .unwrap_or_else(|_| json!({})); + let output = execute_tool(args, repository, task, arm, name, &arguments)?; + write_json( + &arm_dir.join(format!("tool-{turn}-{call_index}.json")), + &json!({"name":name,"arguments":arguments,"output":&output}), + )?; + } + } + Ok(()) +} + +fn tool_schemas(arm: ArmKind, allow_spectra_context: bool) -> Value { + let read = json!({ + "type":"function", "name":"read_source", + "description":"Read a bounded, line-numbered source range after locating an exact file.", + "parameters":{"type":"object","properties":{ + "path":{"type":"string"}, "start":{"type":"integer"}, "end":{"type":"integer"} + },"required":["path","start","end"],"additionalProperties":false}, + "strict":true + }); + if arm == ArmKind::Baseline { + json!([{ + "type":"function", "name":"search_code", + "description":"Literal search across repository source. Use a precise symbol or phrase.", + "parameters":{"type":"object","properties":{"query":{"type":"string"}}, + "required":["query"],"additionalProperties":false}, "strict":true + }, read]) + } else if allow_spectra_context { + json!([{ + "type":"function", "name":"spectra_context", + "description":"Get the single available follow-up context packet only when the initial packet is insufficient. Use read_source for source detail.", + "parameters":{"type":"object","properties":{ + "query":{"type":"string"}, + "intent":{"type":"string","enum":["auto","resume","locate","flow","change","inspect"]} + },"required":["query","intent"],"additionalProperties":false}, "strict":true + }, read]) + } else { + json!([read]) + } +} + +fn execute_tool( + args: &Args, + repository: &Path, + task: &ExpandedTask, + arm: ArmKind, + name: &str, + arguments: &Value, +) -> Result> { + let output = (|| -> Result> { + match name { + "search_code" if arm == ArmKind::Baseline => { + search_code(repository, string_argument(arguments, "query")?) + } + "read_source" => read_source( + repository, + string_argument(arguments, "path")?, + integer_argument(arguments, "start", 1), + integer_argument(arguments, "end", 120), + ), + "spectra_context" if arm == ArmKind::Spectra => spectra_context( + args, + repository, + task, + string_argument(arguments, "query")?, + string_argument(arguments, "intent").unwrap_or("auto"), + "delta", + ), + _ => Ok(format!("tool_error unsupported tool {name}")), + } + })() + .unwrap_or_else(|error| format!("tool_error {name} failed: {error}")); + Ok(bound_text(output, MAX_TOOL_OUTPUT)) +} + +fn search_code(repository: &Path, query: &str) -> Result> { + if query.trim().is_empty() || query.len() > 200 { + return Ok("tool_error search query must contain 1..200 characters".into()); + } + let output = Command::new("rg") + .args([ + "--line-number", + "--fixed-strings", + "--color", + "never", + "--glob", + "!.git/**", + "--glob", + "!.spectra/**", + query, + ".", + ]) + .current_dir(repository) + .output()?; + if !output.status.success() && output.status.code() != Some(1) { + return Ok(format!( + "tool_error search failed: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + let text = String::from_utf8_lossy(&output.stdout); + let lines = text.lines().take(80).collect::>(); + if lines.is_empty() { + Ok("No literal matches.".into()) + } else { + Ok(lines.join("\n")) + } +} + +fn read_source( + repository: &Path, + relative: &str, + start: usize, + end: usize, +) -> Result> { + let relative_path = Path::new(relative.trim_start_matches("./")); + if relative_path.is_absolute() + || relative_path.components().any(|component| { + matches!(component, Component::ParentDir | Component::RootDir | Component::Prefix(_)) + }) + || relative_path.components().any(|component| { + matches!(component, Component::Normal(name) if name == ".git" || name == ".spectra" || name == ".env") + }) + { + return Ok("tool_error path is outside the readable source boundary".into()); + } + let path = repository.join(relative_path).canonicalize()?; + let root = repository.canonicalize()?; + if !path.starts_with(&root) || !path.is_file() { + return Ok("tool_error path is not a repository file".into()); + } + let content = fs::read_to_string(path)?; + let lines = content.lines().collect::>(); + let start = start.max(1); + let end = end + .max(start) + .min(start.saturating_add(199)) + .min(lines.len()); + if start > lines.len() { + return Ok(format!( + "tool_error start {start} is past {} lines", + lines.len() + )); + } + Ok(lines + .iter() + .enumerate() + .take(end) + .skip(start - 1) + .map(|(index, line)| format!("{}\t{line}", index + 1)) + .collect::>() + .join("\n")) +} + +fn spectra_context( + args: &Args, + repository: &Path, + task: &ExpandedTask, + query: &str, + intent: &str, + delivery: &str, +) -> Result> { + let intent = match intent { + "auto" | "resume" | "locate" | "flow" | "change" | "inspect" => intent, + _ => "auto", + }; + let output = Command::new(&args.spectra_bin) + .args([ + "context", + query, + "--path", + repository.to_str().ok_or("non-UTF-8 repository path")?, + "--token-budget", + &args.token_budget.to_string(), + "--intent", + intent, + "--delivery", + delivery, + "--source-harness", + "grok-v04-eval", + "--session-id", + &task.id, + ]) + .output()?; + if !output.status.success() { + return Ok(format!( + "tool_error spectra_context failed: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(String::from_utf8(output.stdout)?) +} + +fn api_response( + key: &str, + model: &str, + history: &[Value], + tools: &Value, + raw_path: &Path, +) -> Result<(Value, u128), Box> { + if raw_path.exists() { + let envelope: Value = serde_json::from_slice(&fs::read(raw_path)?)?; + return Ok(( + envelope["response"].clone(), + envelope["elapsed_ms"].as_u64().unwrap_or(0) as u128, + )); + } + let request = json!({ + "model":model, + "store":false, + "reasoning":{"effort":"low"}, + "max_output_tokens":900, + "include":["reasoning.encrypted_content"], + "tools":tools, + "input":history, + }); + let request_bytes = serde_json::to_vec(&request)?; + let started = Instant::now(); + for attempt in 0..=3_u32 { + let mut child = Command::new("curl") + .args([ + "-sS", + "https://api.x.ai/v1/responses", + "-m", + "3600", + "-H", + "Content-Type: application/json", + "-H", + &format!("Authorization: Bearer {key}"), + "--data-binary", + "@-", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + child + .stdin + .take() + .ok_or("curl stdin unavailable")? + .write_all(&request_bytes)?; + let response = child.wait_with_output()?; + if response.status.success() + && let Ok(value) = serde_json::from_slice::(&response.stdout) + { + if value["error"].is_null() { + let elapsed = started.elapsed().as_millis(); + write_json(raw_path, &json!({"elapsed_ms":elapsed,"response":value}))?; + return Ok((value, elapsed)); + } + if attempt < 3 && retriable_api_error(&value["error"]) { + std::thread::sleep(Duration::from_secs(1_u64 << attempt)); + continue; + } + return Err(format!("xAI API error: {}", value["error"]).into()); + } + if attempt < 3 { + std::thread::sleep(Duration::from_secs(1_u64 << attempt)); + continue; + } + return Err(format!( + "xAI request failed: {}", + String::from_utf8_lossy(&response.stderr) + ) + .into()); + } + unreachable!() +} + +fn add_usage(result: &mut ArmResult, response: &Value) { + let usage = &response["usage"]; + result.input_tokens = result + .input_tokens + .saturating_add(number(usage, "input_tokens")); + result.cached_input_tokens = result + .cached_input_tokens + .saturating_add(number(&usage["input_tokens_details"], "cached_tokens")); + result.output_tokens = result + .output_tokens + .saturating_add(number(usage, "output_tokens")); + result.cost_usd += usage["cost_in_usd_ticks"].as_u64().unwrap_or(0) as f64 / 10_000_000_000.0; +} + +fn account_output(accounting: &mut DeliveryAccounting, output: &str) { + if !accounting.seen.insert(output.to_owned()) { + accounting.repeated_bytes = accounting + .repeated_bytes + .saturating_add(output.len() as u64); + } +} + +fn account_packet(accounting: &mut DeliveryAccounting, output: &str) { + let Some(header) = output.lines().find(|line| line.starts_with("C1 ")) else { + return; + }; + accounting.packets_total += 1; + let budget = header + .split_whitespace() + .find_map(|part| part.strip_prefix("budget=")) + .and_then(|value| value.parse::().ok()); + let used = header + .split_whitespace() + .find_map(|part| part.strip_prefix("used≈")) + .and_then(|value| value.parse::().ok()); + if matches!((budget, used), (Some(budget), Some(used)) if used <= budget) { + accounting.packets_within_budget += 1; + } +} + +fn extract_answer(response: &Value) -> String { + response["output"] + .as_array() + .into_iter() + .flatten() + .filter(|item| item["type"] == "message") + .flat_map(|item| item["content"].as_array().into_iter().flatten()) + .filter(|content| content["type"] == "output_text") + .filter_map(|content| content["text"].as_str()) + .collect::>() + .join("\n") +} + +fn valid_anchor_count(repository: &Path, answer: &str) -> usize { + answer + .split_whitespace() + .filter_map(|token| { + let token = token + .trim_matches(|ch: char| matches!(ch, '`' | ',' | '.' | ')' | '(' | '[' | ']')); + let (path, range) = token.rsplit_once(':')?; + let line = range.split('-').next()?.parse::().ok()?; + let relative = path.trim_start_matches("./"); + if line == 0 || relative.contains("..") || !relative.contains('.') { + return None; + } + let content = fs::read_to_string(repository.join(relative)).ok()?; + (line <= content.lines().count()).then_some(format!("{relative}:{line}")) + }) + .collect::>() + .len() +} + +fn deterministic_success(task: &ExpandedTask, result: &ArmResult) -> bool { + if result.answer.len() < 80 || result.valid_anchors == 0 { + return false; + } + let answer = result.answer.to_ascii_lowercase(); + match task.category.as_str() { + "flow" => result.valid_anchors >= 2, + "change-impact" | "repair" => answer.contains("test") || answer.contains("verify"), + "resume" => answer.contains("next") || answer.contains("verify"), + _ => true, + } +} + +fn write_reports( + args: &Args, + manifest: &Manifest, + details: &[DetailTask], +) -> Result<(), Box> { + let environment_id = format!("{}-local", args.model); + let tasks = details + .iter() + .map(|task| { + let baseline = gate_arm( + &task.baseline, + tool_schema_estimate(ArmKind::Baseline, task.baseline.api_calls), + ); + let spectra = gate_arm( + &task.spectra, + tool_schema_estimate(ArmKind::Spectra, task.spectra.api_calls), + ); + GateTask { + id: task.id.clone(), + repository: task.repository.clone(), + environment_id: environment_id.clone(), + category: task.category.clone(), + packets_within_budget: task.spectra.packets_within_budget, + packets_total: task.spectra.packets_total.max(1), + baseline, + spectra, + } + }) + .collect(); + let gate = GateReport { + schema_version: 1, + environments: vec![Environment { + id: environment_id, + harness: "spectra-v04-grok-eval".into(), + model: args.model.clone(), + }], + tasks, + forbidden_findings: privacy_findings(&args.corpus_root, manifest), + }; + let detail = DetailReport { + schema_version: 1, + label: "single-provider-pilot", + generated_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(), + model: args.model.clone(), + input_schema_accounting: "conservative local estimate; provider reports total and cached input only", + tasks: details.to_vec(), + }; + write_json(&args.output.join("reviewed-v0.4.json"), &gate)?; + write_json(&args.output.join("grok-pilot-details.json"), &detail)?; + Ok(()) +} + +fn gate_arm(result: &ArmResult, schema_estimate: u64) -> GateArm { + let schema = schema_estimate.min(result.input_tokens.saturating_sub(1)); + GateArm { + input_tokens: result.input_tokens, + input_schema_tokens: schema, + input_text_tokens: result.input_tokens.saturating_sub(schema), + image_tokens: 0, + cached_input_tokens: result.cached_input_tokens, + output_tokens: result.output_tokens.max(1), + solved: result.solved, + tool_calls: if result.spectra_calls == 0 { + result.retrieval_calls + } else { + result.spectra_calls + }, + latency_ms: result.latency_ms.max(1), + repeated_context_bytes: result.repeated_context_bytes, + } +} + +fn tool_schema_estimate(arm: ArmKind, calls: u64) -> u64 { + estimate_tokens(&serde_json::to_string(&tool_schemas(arm, true)).unwrap_or_default()) as u64 + * calls +} + +fn privacy_findings(corpus: &Path, manifest: &Manifest) -> Vec { + let mut findings = Vec::new(); + for repository in &manifest.repositories { + for name in ["context-receipts-v1.json", "metrics-v1.json"] { + let path = corpus.join(&repository.id).join(".spectra").join(name); + let Ok(text) = fs::read_to_string(&path) else { + continue; + }; + if text.contains("grok-v04-eval") + || manifest + .task_templates + .iter() + .any(|task| text.contains(&task.prompt)) + { + findings.push(format!( + "{} contains raw session or prompt text", + path.display() + )); + } + if text.to_ascii_lowercase().contains("xai_key") || text.contains("Bearer ") { + findings.push(format!( + "{} contains credential-shaped text", + path.display() + )); + } + } + } + findings +} + +fn summary(details: &[DetailTask]) -> (f64, f64) { + let baseline = details + .iter() + .map(|task| task.baseline.input_tokens) + .sum::() as f64; + let spectra = details + .iter() + .map(|task| task.spectra.input_tokens) + .sum::() as f64; + let cost = details + .iter() + .map(|task| task.baseline.cost_usd + task.spectra.cost_usd) + .sum(); + ( + if baseline == 0.0 { + 0.0 + } else { + 1.0 - spectra / baseline + }, + cost, + ) +} + +fn repository_commit<'a>( + manifest: &'a Manifest, + id: &str, +) -> Result<&'a str, Box> { + manifest + .repositories + .iter() + .find(|repository| repository.id == id) + .map(|repository| repository.commit.as_str()) + .ok_or_else(|| format!("missing repository {id}").into()) +} + +fn verify_checkout(path: &Path, expected: &str) -> Result<(), Box> { + let output = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(path) + .output()?; + let actual = String::from_utf8(output.stdout)?; + if !output.status.success() || actual.trim() != expected { + return Err(format!( + "{} is at {}, expected {expected}", + path.display(), + actual.trim() + ) + .into()); + } + Ok(()) +} + +fn api_key(path: &Path) -> Result> { + if let Ok(value) = std::env::var("XAI_KEY") + && !value.trim().is_empty() + { + return Ok(value); + } + for line in fs::read_to_string(path)?.lines() { + let line = line.trim().strip_prefix("export ").unwrap_or(line.trim()); + if let Some(value) = line.strip_prefix("XAI_KEY=") { + let value = value.trim().trim_matches(['\'', '"']); + if !value.is_empty() { + return Ok(value.into()); + } + } + } + Err(format!( + "XAI_KEY was not found in the environment or {}", + path.display() + ) + .into()) +} + +fn retriable_api_error(error: &Value) -> bool { + let code = error["code"].as_str().unwrap_or_default(); + let message = error["message"].as_str().unwrap_or_default(); + matches!(code, "rate_limit_exceeded" | "server_error") + || message.contains("rate limit") + || message.contains("temporarily unavailable") +} + +fn write_json(path: &Path, value: &impl Serialize) -> Result<(), Box> { + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, serde_json::to_vec_pretty(value)?)?; + fs::rename(temporary, path)?; + Ok(()) +} + +fn bound_text(mut value: String, max: usize) -> String { + if value.len() <= max { + return value; + } + while !value.is_char_boundary(max.min(value.len())) { + value.pop(); + } + value.truncate(max); + value.push_str("\n[tool output bounded]"); + value +} + +fn string_argument<'a>(value: &'a Value, key: &str) -> Result<&'a str, Box> { + value[key] + .as_str() + .ok_or_else(|| format!("missing string argument {key}").into()) +} + +fn integer_argument(value: &Value, key: &str, default: usize) -> usize { + value[key] + .as_u64() + .and_then(|number| usize::try_from(number).ok()) + .unwrap_or(default) +} + +fn number(value: &Value, key: &str) -> u64 { + value[key].as_u64().unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expands_the_frozen_matrix_and_filters_without_changing_order() { + let manifest: Manifest = + serde_json::from_str(include_str!("../../../../benchmarks/v0.4-holdout.json")).unwrap(); + let args = Args::parse_from([ + "test", + "--corpus-root", + "/tmp/corpus", + "--output", + "/tmp/output", + ]); + let tasks = expand_tasks(&manifest, &args); + assert_eq!(tasks.len(), 100); + assert_eq!(tasks[0].id, "ripgrep-navigation"); + assert_eq!(tasks[99].id, "fmt-resume"); + } + + #[test] + fn packet_accounting_reads_context_headers() { + let mut accounting = DeliveryAccounting::default(); + account_packet( + &mut accounting, + "C1 id=p1 intent=flow index=v4 budget=600 used≈438 delivery=full", + ); + assert_eq!(accounting.packets_total, 1); + assert_eq!(accounting.packets_within_budget, 1); + } + + #[test] + fn source_reader_rejects_parent_traversal() { + let root = std::env::temp_dir(); + assert!( + read_source(&root, "../secret", 1, 2) + .unwrap() + .contains("outside") + ); + } + + #[test] + fn ordinary_missing_source_is_returned_to_the_model_as_a_tool_error() { + let args = Args::parse_from([ + "test", + "--corpus-root", + "/tmp/corpus", + "--output", + "/tmp/output", + ]); + let task = ExpandedTask { + id: "task".into(), + repository: "repo".into(), + category: "navigation".into(), + prompt: "locate".into(), + }; + let output = execute_tool( + &args, + &std::env::temp_dir(), + &task, + ArmKind::Baseline, + "read_source", + &json!({"path":"definitely-missing.rs","start":1,"end":10}), + ) + .unwrap(); + assert!(output.starts_with("tool_error read_source failed:")); + } + + #[test] + fn spectra_context_schema_is_removed_after_the_call_ceiling() { + let available = tool_schemas(ArmKind::Spectra, true).to_string(); + let exhausted = tool_schemas(ArmKind::Spectra, false).to_string(); + assert!(available.contains("spectra_context")); + assert!(!exhausted.contains("spectra_context")); + assert!(exhausted.contains("read_source")); + assert_eq!(MAX_SPECTRA_CONTEXT_CALLS, 2); + } +} diff --git a/crates/spectra-cli/src/context_state.rs b/crates/spectra-cli/src/context_state.rs new file mode 100644 index 0000000..2563a24 --- /dev/null +++ b/crates/spectra-cli/src/context_state.rs @@ -0,0 +1,561 @@ +use std::{ + collections::BTreeMap, + fs::{self, OpenOptions}, + io::Write, + path::Path, + thread, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use atomic_write_file::AtomicWriteFile; +use serde::{Deserialize, Serialize}; +use spectra_core::{EvidenceRecord, LedgerSource, PackedEvidence, pack_evidence}; + +const RECEIPTS_PATH: &str = ".spectra/context-receipts-v1.json"; +const METRICS_PATH: &str = ".spectra/metrics-v1.json"; +const LOCK_PATH: &str = ".spectra/context-runtime-v1.lock"; +const MAX_SESSIONS: usize = 128; +const MAX_EVIDENCE: usize = 256; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Delivery { + Delta, + Full, +} + +impl Delivery { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Delta => "delta", + Self::Full => "full", + } + } +} + +#[derive(Debug)] +pub(crate) struct DeliveryResult { + pub packed: PackedEvidence, + pub duplicate_evidence: usize, + pub effective_delivery: Delivery, +} + +pub(crate) struct DeliveryRequest<'a> { + pub source: Option<&'a LedgerSource>, + pub requested: Delivery, + pub token_budget: usize, + pub offset: usize, + pub index_version: u32, + pub ledger_sequence: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ReceiptFile { + version: u32, + salt: String, + clock: u64, + sessions: BTreeMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ReceiptSession { + last_access: u64, + index_version: u32, + ledger_sequence: u64, + evidence: BTreeMap, +} + +pub(crate) fn deliver( + project: &Path, + records: Vec, + request: DeliveryRequest<'_>, +) -> DeliveryResult { + let Some(source) = request.source else { + return DeliveryResult { + packed: pack_evidence(records, request.token_budget, request.offset), + duplicate_evidence: 0, + effective_delivery: Delivery::Full, + }; + }; + let fallback = records.clone(); + transaction(project, |receipts, recovered| { + let effective = if recovered { + Delivery::Full + } else { + request.requested + }; + if receipts.salt.is_empty() { + receipts.version = 1; + receipts.salt = fresh_salt(project); + } + receipts.clock = receipts.clock.saturating_add(1); + let clock = receipts.clock; + let key = digest( + &receipts.salt, + &format!("{}\0{}", source.harness, source.session_id), + ); + let session = receipts.sessions.entry(key).or_default(); + session.last_access = clock; + session.index_version = request.index_version; + session.ledger_sequence = request.ledger_sequence; + if effective == Delivery::Full { + session.evidence.clear(); + } + let duplicate_evidence = if effective == Delivery::Delta { + records + .iter() + .filter(|record| session.evidence.contains_key(&record.id)) + .count() + } else { + 0 + }; + let candidates = records + .into_iter() + .filter(|record| { + effective == Delivery::Full || !session.evidence.contains_key(&record.id) + }) + .collect::>(); + let effective_offset = if effective == Delivery::Delta { + 0 + } else { + request.offset + }; + let mut packed = pack_evidence(candidates, request.token_budget, effective_offset); + for record in &packed.records { + session.evidence.insert(record.id.clone(), clock); + } + trim_evidence(session); + trim_sessions(receipts); + if effective == Delivery::Delta && packed.next_offset.is_some() { + packed.next_offset = Some(0); + } + DeliveryResult { + packed, + duplicate_evidence, + effective_delivery: effective, + } + }) + .unwrap_or_else(|_| DeliveryResult { + packed: pack_evidence(fallback, request.token_budget, request.offset), + duplicate_evidence: 0, + effective_delivery: Delivery::Full, + }) +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct Metrics { + pub version: u32, + pub calls: u64, + pub estimated_tokens_emitted: u64, + pub duplicate_evidence_avoided: u64, + pub maps_requested: u64, + pub errors: u64, + pub full_deliveries: u64, + pub delta_deliveries: u64, + pub latency_lt_10_ms: u64, + pub latency_lt_100_ms: u64, + pub latency_lt_1_s: u64, + pub latency_gte_1_s: u64, + pub calls_by_intent: BTreeMap, +} + +pub(crate) struct MetricSample<'a> { + pub intent: &'a str, + pub estimated_tokens: usize, + pub duplicates: usize, + pub map: bool, + pub error: bool, + pub delivery: Delivery, + pub elapsed: Duration, +} + +pub(crate) fn record_metrics(project: &Path, sample: MetricSample<'_>) { + if std::env::var("SPECTRA_METRICS") + .ok() + .is_some_and(|value| value.eq_ignore_ascii_case("off")) + { + return; + } + let _ = with_lock(project, || { + let path = project.join(METRICS_PATH); + let mut metrics = read_json::(&path) + .ok() + .flatten() + .unwrap_or_default(); + metrics.version = 1; + metrics.calls += 1; + metrics.estimated_tokens_emitted += sample.estimated_tokens as u64; + metrics.duplicate_evidence_avoided += sample.duplicates as u64; + metrics.maps_requested += u64::from(sample.map); + metrics.errors += u64::from(sample.error); + match sample.delivery { + Delivery::Full => metrics.full_deliveries += 1, + Delivery::Delta => metrics.delta_deliveries += 1, + } + *metrics + .calls_by_intent + .entry(sample.intent.to_owned()) + .or_default() += 1; + match sample.elapsed.as_millis() { + 0..=9 => metrics.latency_lt_10_ms += 1, + 10..=99 => metrics.latency_lt_100_ms += 1, + 100..=999 => metrics.latency_lt_1_s += 1, + _ => metrics.latency_gte_1_s += 1, + } + write_json(&path, &metrics) + }); +} + +pub(crate) fn record_error(project: &Path) { + if std::env::var("SPECTRA_METRICS") + .ok() + .is_some_and(|value| value.eq_ignore_ascii_case("off")) + { + return; + } + let _ = with_lock(project, || { + let path = project.join(METRICS_PATH); + let mut metrics = read_json::(&path) + .ok() + .flatten() + .unwrap_or_default(); + metrics.version = 1; + metrics.errors += 1; + write_json(&path, &metrics) + }); +} + +pub(crate) fn read_metrics(project: &Path) -> Result> { + Ok(read_json(&project.join(METRICS_PATH))?.unwrap_or_default()) +} + +pub(crate) fn reset_metrics(project: &Path) -> Result<(), Box> { + with_lock(project, || { + let path = project.join(METRICS_PATH); + if path.exists() { + fs::remove_file(path)?; + } + Ok(()) + }) +} + +fn transaction( + project: &Path, + operation: impl FnOnce(&mut ReceiptFile, bool) -> T, +) -> Result> { + with_lock(project, || { + let path = project.join(RECEIPTS_PATH); + let (mut receipts, recovered) = match read_json::(&path) { + Ok(receipts) => (receipts.unwrap_or_default(), false), + Err(_) => (ReceiptFile::default(), true), + }; + let result = operation(&mut receipts, recovered); + write_json(&path, &receipts)?; + Ok(result) + }) +} + +fn with_lock( + project: &Path, + operation: impl FnOnce() -> Result>, +) -> Result> { + let path = project.join(LOCK_PATH); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut acquired = false; + for _ in 0..1_000 { + match OpenOptions::new().create_new(true).write(true).open(&path) { + Ok(mut file) => { + writeln!(file, "{}", std::process::id())?; + acquired = true; + break; + } + Err(error) if is_lock_contention(&error) => { + let stale = fs::metadata(&path) + .and_then(|metadata| metadata.modified()) + .and_then(|modified| modified.elapsed().map_err(std::io::Error::other)) + .is_ok_and(|age| age > Duration::from_secs(30)); + if stale { + let _ = fs::remove_file(&path); + } else { + thread::sleep(Duration::from_millis(2)); + } + } + Err(error) => return Err(error.into()), + } + } + if !acquired { + return Err("timed out acquiring context runtime lock".into()); + } + let result = operation(); + let unlock = fs::remove_file(&path); + match (result, unlock) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error), + (_, Err(error)) => Err(error.into()), + } +} + +fn is_lock_contention(error: &std::io::Error) -> bool { + if error.kind() == std::io::ErrorKind::AlreadyExists { + return true; + } + + // Windows can report a lock file being deleted by the previous owner as + // access denied, sharing violation, or lock violation. Retrying keeps an + // ordinary handoff from becoming a fail-open receipt write. + #[cfg(windows)] + { + matches!(error.raw_os_error(), Some(5 | 32 | 33)) + } + #[cfg(not(windows))] + { + false + } +} + +fn read_json Deserialize<'de>>( + path: &Path, +) -> Result, Box> { + if !path.exists() { + return Ok(None); + } + Ok(Some(serde_json::from_slice(&fs::read(path)?)?)) +} + +fn write_json(path: &Path, value: &T) -> Result<(), Box> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut encoded = serde_json::to_vec(value)?; + encoded.push(b'\n'); + let mut file = AtomicWriteFile::open(path)?; + file.write_all(&encoded)?; + file.commit()?; + Ok(()) +} + +fn trim_evidence(session: &mut ReceiptSession) { + while session.evidence.len() > MAX_EVIDENCE { + let oldest = session + .evidence + .iter() + .min_by_key(|(id, clock)| (*clock, *id)) + .map(|(id, _)| id.clone()); + if let Some(oldest) = oldest { + session.evidence.remove(&oldest); + } + } +} + +fn trim_sessions(receipts: &mut ReceiptFile) { + while receipts.sessions.len() > MAX_SESSIONS { + let oldest = receipts + .sessions + .iter() + .min_by_key(|(key, session)| (session.last_access, *key)) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + receipts.sessions.remove(&oldest); + } + } +} + +fn fresh_salt(project: &Path) -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + digest( + "spectra-v1", + &format!("{}\0{}\0{nanos}", project.display(), std::process::id()), + ) +} + +pub(crate) fn digest(salt: &str, value: &str) -> String { + fn hash(seed: u64, bytes: impl Iterator) -> u64 { + bytes.fold(seed, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3) + }) + } + let bytes = salt.bytes().chain([0]).chain(value.bytes()); + let first = hash(0xcbf29ce484222325, bytes.clone()); + let second = hash(0x84222325cbf29ce4, bytes); + format!("{first:016x}{second:016x}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn temp_project(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "spectra-context-state-{label}-{}", + fresh_salt(Path::new(label)) + )); + fs::create_dir_all(&path).unwrap(); + path + } + + fn evidence(value: &str) -> EvidenceRecord { + EvidenceRecord { + id: digest("evidence", value), + priority: 1, + text: value.into(), + shrink_lines: false, + } + } + + fn request(source: &LedgerSource, requested: Delivery) -> DeliveryRequest<'_> { + DeliveryRequest { + source: Some(source), + requested, + token_budget: 100, + offset: 0, + index_version: 4, + ledger_sequence: 1, + } + } + + #[test] + fn receipts_are_session_isolated_and_store_no_raw_session_or_body() { + let project = temp_project("isolation"); + let first = LedgerSource { + harness: "codex".into(), + session_id: "raw-session-one".into(), + }; + let second = LedgerSource { + harness: "codex".into(), + session_id: "raw-session-two".into(), + }; + let records = vec![evidence("private source body")]; + let one = deliver(&project, records.clone(), request(&first, Delivery::Delta)); + assert_eq!(one.packed.records.len(), 1); + let duplicate = deliver(&project, records.clone(), request(&first, Delivery::Delta)); + assert!(duplicate.packed.records.is_empty()); + assert_eq!(duplicate.duplicate_evidence, 1); + let other = deliver(&project, records, request(&second, Delivery::Delta)); + assert_eq!(other.packed.records.len(), 1); + let persisted = fs::read_to_string(project.join(RECEIPTS_PATH)).unwrap(); + assert!(!persisted.contains("raw-session")); + assert!(!persisted.contains("private source body")); + fs::remove_dir_all(project).unwrap(); + } + + #[test] + fn full_delivery_resets_the_receipt_baseline() { + let project = temp_project("full"); + let source = LedgerSource { + harness: "custom".into(), + session_id: "s1".into(), + }; + let records = vec![evidence("anchor")]; + deliver(&project, records.clone(), request(&source, Delivery::Delta)); + let full = deliver(&project, records, request(&source, Delivery::Full)); + assert_eq!(full.packed.records.len(), 1); + assert_eq!(full.effective_delivery, Delivery::Full); + fs::remove_dir_all(project).unwrap(); + } + + #[test] + fn metrics_can_be_recorded_read_and_reset() { + let project = temp_project("metrics"); + record_metrics( + &project, + MetricSample { + intent: "locate", + estimated_tokens: 42, + duplicates: 3, + map: false, + error: false, + delivery: Delivery::Delta, + elapsed: Duration::from_millis(12), + }, + ); + let metrics = read_metrics(&project).unwrap(); + assert_eq!(metrics.calls, 1); + assert_eq!(metrics.duplicate_evidence_avoided, 3); + assert_eq!(metrics.latency_lt_100_ms, 1); + reset_metrics(&project).unwrap(); + assert_eq!(read_metrics(&project).unwrap().calls, 0); + fs::remove_dir_all(project).unwrap(); + } + + #[test] + fn corrupt_receipts_fail_open_to_full_and_recover() { + let project = temp_project("corrupt"); + let path = project.join(RECEIPTS_PATH); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"not json").unwrap(); + let source = LedgerSource { + harness: "custom".into(), + session_id: "s1".into(), + }; + let result = deliver( + &project, + vec![evidence("anchor")], + request(&source, Delivery::Delta), + ); + assert_eq!(result.effective_delivery, Delivery::Full); + assert_eq!(result.packed.records.len(), 1); + assert!(serde_json::from_slice::(&fs::read(path).unwrap()).is_ok()); + fs::remove_dir_all(project).unwrap(); + } + + #[test] + fn concurrent_receipt_writers_preserve_each_delivery() { + let project = temp_project("concurrent"); + let source = LedgerSource { + harness: "custom".into(), + session_id: "s1".into(), + }; + let handles = (0..8) + .map(|index| { + let project = project.clone(); + let source = source.clone(); + std::thread::spawn(move || { + deliver( + &project, + vec![evidence(&format!("anchor-{index}"))], + request(&source, Delivery::Delta), + ) + }) + }) + .collect::>(); + for handle in handles { + assert_eq!(handle.join().unwrap().packed.records.len(), 1); + } + let receipts: ReceiptFile = + serde_json::from_slice(&fs::read(project.join(RECEIPTS_PATH)).unwrap()).unwrap(); + assert_eq!(receipts.sessions.values().next().unwrap().evidence.len(), 8); + fs::remove_dir_all(project).unwrap(); + } + + #[test] + fn receipt_storage_enforces_session_and_evidence_caps() { + let mut session = ReceiptSession::default(); + for index in 0..(MAX_EVIDENCE + 10) { + session + .evidence + .insert(format!("e{index:03}"), index as u64); + } + trim_evidence(&mut session); + assert_eq!(session.evidence.len(), MAX_EVIDENCE); + assert!(!session.evidence.contains_key("e000")); + + let mut receipts = ReceiptFile::default(); + for index in 0..(MAX_SESSIONS + 10) { + receipts.sessions.insert( + format!("s{index:03}"), + ReceiptSession { + last_access: index as u64, + ..ReceiptSession::default() + }, + ); + } + trim_sessions(&mut receipts); + assert_eq!(receipts.sessions.len(), MAX_SESSIONS); + assert!(!receipts.sessions.contains_key("s000")); + } +} diff --git a/crates/spectra-cli/src/main.rs b/crates/spectra-cli/src/main.rs index 74cd79e..4bda057 100644 --- a/crates/spectra-cli/src/main.rs +++ b/crates/spectra-cli/src/main.rs @@ -1,5 +1,6 @@ mod agents; mod autosync; +mod context_state; mod git_sync; mod hook; mod install; @@ -15,9 +16,14 @@ use std::{ use agents::{Agent, Location}; use clap::{Parser, Subcommand, ValueEnum}; -use spectra_core::{CodeIndex, INDEX_VERSION, IndexReport, sync_project}; +use spectra_core::{CodeIndex, INDEX_VERSION, IndexReport, LedgerSource, sync_project}; use spectra_render::{MapArtifact, map_project}; +use crate::{ + context_state::Delivery, + mcp_query::{ContextIntent, ContextOptions}, +}; + #[derive(Debug, Parser)] #[command( name = "spectra", @@ -119,6 +125,35 @@ enum Command { #[arg(long)] out: Option, }, + /// Produce one budgeted adaptive Context Packet v1. + Context { + query: String, + #[arg(long, default_value = ".")] + path: PathBuf, + #[arg(long, default_value_t = 600, value_parser = clap::value_parser!(u16).range(128..=2000))] + token_budget: u16, + #[arg(long, value_enum, default_value_t = CliContextIntent::Auto)] + intent: CliContextIntent, + #[arg(long, value_enum, default_value_t = CliRepresentation::Text)] + representation: CliRepresentation, + #[arg(long, value_enum, default_value_t = CliDelivery::Delta)] + delivery: CliDelivery, + #[arg(long, requires = "session_id")] + source_harness: Option, + #[arg(long, requires = "source_harness")] + session_id: Option, + #[arg(long)] + cursor: Option, + }, + /// Show or reset privacy-safe local context efficiency counters. + Stats { + #[arg(long, default_value = ".")] + path: PathBuf, + #[arg(long)] + json: bool, + #[arg(long)] + reset: bool, + }, /// Run Spectra's MCP server over stdio. Serve { #[arg(long)] @@ -129,6 +164,53 @@ enum Command { }, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] +enum CliContextIntent { + #[default] + Auto, + Resume, + Locate, + Flow, + Change, + Inspect, +} + +impl From for ContextIntent { + fn from(value: CliContextIntent) -> Self { + match value { + CliContextIntent::Auto => Self::Auto, + CliContextIntent::Resume => Self::Resume, + CliContextIntent::Locate => Self::Locate, + CliContextIntent::Flow => Self::Flow, + CliContextIntent::Change => Self::Change, + CliContextIntent::Inspect => Self::Inspect, + } + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] +enum CliRepresentation { + #[default] + Text, + Map, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] +enum CliDelivery { + #[default] + Delta, + Full, +} + +impl From for Delivery { + fn from(value: CliDelivery) -> Self { + match value { + CliDelivery::Delta => Self::Delta, + CliDelivery::Full => Self::Full, + } + } +} + #[derive(Debug, Subcommand)] enum AutosyncCommand { /// Install ownership-safe post-commit, post-merge, and post-checkout hooks. @@ -290,6 +372,68 @@ async fn run(cli: Cli) -> Result<(), Box> { println!("SVG {}", display_relative(&artifact.svg_path, &path)); print_anchors(&artifact); } + Command::Context { + query, + path, + token_budget, + intent, + representation, + delivery, + source_harness, + session_id, + cursor, + } => { + let source = source_harness + .zip(session_id) + .map(|(harness, session_id)| LedgerSource { + harness, + session_id, + }); + let autosync = autosync::AutoSync::default(); + let view = mcp_query::open_project(&autosync, path.to_str())?; + let map_requested = representation == CliRepresentation::Map; + let packet = mcp_query::context_packet( + &view, + ContextOptions { + query: &query, + token_budget: usize::from(token_budget), + intent: intent.into(), + delivery: delivery.into(), + source, + cursor: cursor.as_deref(), + map_requested, + }, + )?; + println!("{}", packet.text); + if map_requested { + let output = view.root.join(".spectra/artifacts"); + let artifact = map_project(&view.root, &query, 48, &output)?; + println!("PNG {}", display_relative(&artifact.png_path, &view.root)); + println!("SVG {}", display_relative(&artifact.svg_path, &view.root)); + } + } + Command::Stats { path, json, reset } => { + if reset { + context_state::reset_metrics(&path)?; + } + let metrics = context_state::read_metrics(&path)?; + if json { + println!("{}", serde_json::to_string_pretty(&metrics)?); + } else if reset { + println!("Spectra context metrics reset."); + } else { + println!( + "Spectra Context Stats\ncalls={} emitted_estimated_tokens={} duplicate_evidence_avoided={} maps_requested={} errors={} full={} delta={}", + metrics.calls, + metrics.estimated_tokens_emitted, + metrics.duplicate_evidence_avoided, + metrics.maps_requested, + metrics.errors, + metrics.full_deliveries, + metrics.delta_deliveries + ); + } + } Command::Serve { mcp: true, path: Some(path), diff --git a/crates/spectra-cli/src/mcp.rs b/crates/spectra-cli/src/mcp.rs index 64ae8b2..7e7ec58 100644 --- a/crates/spectra-cli/src/mcp.rs +++ b/crates/spectra-cli/src/mcp.rs @@ -18,9 +18,10 @@ use spectra_core::LedgerSource; use spectra_render::{MapArtifact, map_project}; use crate::autosync::{AutoSync, SyncSnapshot}; +use crate::context_state::{self, Delivery}; use crate::mcp_query::{ - self, BriefOptions, ChangeOptions, Direction, FileFormat, NodeViewOptions, PathMode, - PathOptions, + self, BriefOptions, ChangeOptions, ContextIntent, ContextOptions, Direction, FileFormat, + NodeViewOptions, PathMode, PathOptions, }; #[derive(Clone, Default)] @@ -92,6 +93,79 @@ struct BriefRequest { source: Option, } +#[derive(Clone, Copy, Debug, Default, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum ContextIntentRequest { + #[default] + Auto, + Resume, + Locate, + Flow, + Change, + Inspect, +} + +impl From for ContextIntent { + fn from(value: ContextIntentRequest) -> Self { + match value { + ContextIntentRequest::Auto => Self::Auto, + ContextIntentRequest::Resume => Self::Resume, + ContextIntentRequest::Locate => Self::Locate, + ContextIntentRequest::Flow => Self::Flow, + ContextIntentRequest::Change => Self::Change, + ContextIntentRequest::Inspect => Self::Inspect, + } + } +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum ContextRepresentationRequest { + #[default] + Text, + Map, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum ContextDeliveryRequest { + #[default] + Delta, + Full, +} + +impl From for Delivery { + fn from(value: ContextDeliveryRequest) -> Self { + match value { + ContextDeliveryRequest::Delta => Self::Delta, + ContextDeliveryRequest::Full => Self::Full, + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema)] +struct ContextRequest { + /// Current coding goal or code-navigation question. + query: String, + /// Absolute project path, or any directory inside the project. + #[serde(rename = "projectPath", alias = "project_path")] + project_path: Option, + /// Total text budget in estimated tokens. Defaults to 600. + #[serde(rename = "tokenBudget", alias = "token_budget")] + #[schemars(range(min = 128, max = 2000))] + token_budget: Option, + /// Deterministic retrieval route. Defaults to automatic classification. + intent: Option, + /// Text by default; map must be requested explicitly. + representation: Option, + /// Delta for a supplied session, otherwise a safe full packet. + delivery: Option, + /// Optional exact lifecycle session used for continuity and deduplication. + source: Option, + /// Opaque continuation cursor returned by a previous packet. + cursor: Option, +} + #[derive(Clone, Debug, Default, Deserialize, JsonSchema)] struct ChangesRequest { /// Absolute project path, or any directory inside the project. @@ -287,6 +361,90 @@ struct FilesRequest { #[tool_router] impl SpectraServer { + #[tool( + name = "spectra_context", + description = "Return the smallest sufficient, budgeted code context for the next decision. Text-only unless representation=map; exact sessions receive durable delta delivery.", + annotations( + read_only_hint = false, + destructive_hint = false, + idempotent_hint = false, + open_world_hint = false + ) + )] + async fn spectra_context( + &self, + Parameters(request): Parameters, + ) -> CallToolResult { + if let Some(error) = validate_required(&request.query, "query") { + return error; + } + if request + .cursor + .as_ref() + .is_some_and(|cursor| cursor.len() > 2_048) + { + return CallToolResult::error(vec![ContentBlock::text( + "cursor exceeds 2048 characters", + )]); + } + let source = match parse_source(request.source) { + Ok(source) => source, + Err(error) => return error, + }; + let map_requested = matches!( + request.representation, + Some(ContextRepresentationRequest::Map) + ); + let view = match mcp_query::open_project(&self.autosync, request.project_path.as_deref()) { + Ok(view) => view, + Err(error) => { + return CallToolResult::error(vec![ContentBlock::text(format!( + "spectra_context could not open an indexed project: {error}" + ))]); + } + }; + let packet = match mcp_query::context_packet( + &view, + ContextOptions { + query: &request.query, + token_budget: usize::from(request.token_budget.unwrap_or(600)), + intent: request.intent.unwrap_or_default().into(), + delivery: request.delivery.unwrap_or_default().into(), + source, + cursor: request.cursor.as_deref(), + map_requested, + }, + ) { + Ok(packet) => packet, + Err(error) => { + context_state::record_error(&view.root); + return CallToolResult::error(vec![ContentBlock::text(format!( + "spectra_context failed: {error}" + ))]); + } + }; + if !map_requested { + return CallToolResult::success(vec![ContentBlock::text(packet.text)]); + } + let output = view.root.join(".spectra/artifacts"); + match map_project(&view.root, &request.query, 48, &output).and_then(|artifact| { + fs::read(&artifact.png_path) + .map(|png| (artifact, png)) + .map_err(Into::into) + }) { + Ok((_artifact, png)) => CallToolResult::success(vec![ + ContentBlock::image(STANDARD.encode(png), "image/png"), + ContentBlock::text(packet.text), + ]), + Err(error) => { + context_state::record_error(&view.root); + CallToolResult::error(vec![ContentBlock::text(format!( + "spectra_context map failed: {error}" + ))]) + } + } + } + #[tool( name = "spectra_map", description = "Render a compact PNG code-topology map for a polyglot architecture question. Returns an image plus exact file/line anchors, never source bodies.", @@ -703,8 +861,8 @@ impl SpectraServer { #[tool_handler( name = "spectra", - version = "0.3.0", - instructions = "Use spectra_brief to start or resume work with bounded continuity and ranked anchors. Use spectra_map for visual architecture questions. Change impact, typed paths, bounded source exploration, targeted search, node, caller/callee, file-tree, and status tools are available through SPECTRA_MCP_TOOLS." + version = "0.4.0", + instructions = "Use spectra_context for budgeted, text-first code context. Request representation=map explicitly when a visual topology is worth provider image tokens. Legacy focused tools remain available through SPECTRA_MCP_TOOLS." )] impl ServerHandler for SpectraServer { async fn list_tools( @@ -761,6 +919,24 @@ fn validate_optional_path(value: Option<&str>, field: &str) -> Option, +) -> Result, CallToolResult> { + let Some(source) = source else { + return Ok(None); + }; + if let Some(error) = validate_required(&source.harness, "source.harness") { + return Err(error); + } + if let Some(error) = validate_required(&source.session_id, "source.sessionId") { + return Err(error); + } + Ok(Some(LedgerSource { + harness: source.harness, + session_id: source.session_id, + })) +} + fn relationship_result( autosync: &AutoSync, request: RelationshipRequest, @@ -809,9 +985,7 @@ fn allowed_tools(raw: Option<&str>) -> BTreeSet { .map(|tool| tool.name.into_owned()) .collect::>(); let Some(raw) = raw.filter(|raw| !raw.trim().is_empty()) else { - return ["spectra_brief".to_owned(), "spectra_map".to_owned()] - .into_iter() - .collect(); + return ["spectra_context".to_owned()].into_iter().collect(); }; if raw.trim() == "all" { return all; @@ -912,7 +1086,7 @@ mod tests { } #[test] - fn server_pins_the_codegraph_parity_tool_contract() { + fn server_pins_the_v04_and_legacy_tool_contracts() { let server = SpectraServer::default(); assert_eq!(server.get_info().server_info.name, "spectra"); assert_eq!( @@ -927,6 +1101,7 @@ mod tests { assert_eq!( names, [ + "spectra_context", "spectra_map", "spectra_brief", "spectra_explore", @@ -943,10 +1118,7 @@ mod tests { .into_iter() .collect() ); - assert_eq!( - allowed_tools(None), - ["spectra_brief".to_owned(), "spectra_map".to_owned()].into() - ); + assert_eq!(allowed_tools(None), ["spectra_context".to_owned()].into()); assert_eq!( allowed_tools(Some("explore,node,status")), ["spectra_explore", "spectra_node", "spectra_status"] @@ -954,8 +1126,11 @@ mod tests { .map(str::to_owned) .collect() ); - assert_eq!(allowed_tools(Some("all")).len(), 12); - for tool in tools.iter().filter(|tool| tool.name != "spectra_map") { + assert_eq!(allowed_tools(Some("all")).len(), 13); + for tool in tools + .iter() + .filter(|tool| !matches!(tool.name.as_ref(), "spectra_map" | "spectra_context")) + { let annotations = tool.annotations.as_ref().expect("read tool annotations"); assert_eq!(annotations.read_only_hint, Some(true)); assert_eq!(annotations.idempotent_hint, Some(true)); @@ -963,6 +1138,19 @@ mod tests { assert!(schema["properties"].get("projectPath").is_some()); } let expected_properties = [ + ( + "spectra_context", + &[ + "query", + "projectPath", + "tokenBudget", + "intent", + "representation", + "delivery", + "source", + "cursor", + ][..], + ), ( "spectra_brief", &["query", "projectPath", "tokenBudget", "detail", "source"][..], @@ -1061,6 +1249,21 @@ mod tests { assert_eq!(brief.project_path.as_deref(), Some("/tmp/project")); assert_eq!(brief.token_budget, Some(512)); assert_eq!(brief.source.unwrap().session_id, "s1"); + let context: ContextRequest = serde_json::from_value(serde_json::json!({ + "query":"resume", + "project_path":"/tmp/project", + "token_budget":512, + "representation":"map", + "delivery":"full", + "source":{"harness":"custom","session_id":"s1"} + })) + .unwrap(); + assert_eq!(context.project_path.as_deref(), Some("/tmp/project")); + assert_eq!(context.token_budget, Some(512)); + assert!(matches!( + context.representation, + Some(ContextRepresentationRequest::Map) + )); let changes: ChangesRequest = serde_json::from_value(serde_json::json!({ "project_path":"/tmp/project", "include_tests":false, diff --git a/crates/spectra-cli/src/mcp_query.rs b/crates/spectra-cli/src/mcp_query.rs index 4711e3e..273c309 100644 --- a/crates/spectra-cli/src/mcp_query.rs +++ b/crates/spectra-cli/src/mcp_query.rs @@ -3,16 +3,23 @@ use std::{ fs, path::{Path, PathBuf}, process::Command, + time::Instant, }; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; use spectra_core::ledger::redact_text; use spectra_core::{ - CodeIndex, IndexReport, LedgerEventKind, LedgerSource, LedgerStore, SelectionOptions, + CodeIndex, EvidenceRecord, IndexReport, LedgerEventKind, LedgerSource, LedgerStore, + SelectionOptions, estimate_tokens, graph::{EdgeId, NodeId}, select_subgraph, supported_languages, }; -use crate::autosync::{AutoSync, SyncSnapshot}; +use crate::{ + autosync::{AutoSync, SyncSnapshot}, + context_state::{self, Delivery}, +}; const MAX_TEXT: usize = 24_000; const MAX_SOURCE_LINES: usize = 2_000; @@ -54,6 +61,44 @@ pub(crate) struct BriefOptions<'a> { pub(crate) source: Option, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ContextIntent { + Auto, + Resume, + Locate, + Flow, + Change, + Inspect, +} + +impl ContextIntent { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Resume => "resume", + Self::Locate => "locate", + Self::Flow => "flow", + Self::Change => "change", + Self::Inspect => "inspect", + } + } +} + +pub(crate) struct ContextOptions<'a> { + pub(crate) query: &'a str, + pub(crate) token_budget: usize, + pub(crate) intent: ContextIntent, + pub(crate) delivery: Delivery, + pub(crate) source: Option, + pub(crate) cursor: Option<&'a str>, + pub(crate) map_requested: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct ContextPacket { + pub(crate) text: String, +} + pub(crate) struct ChangeOptions<'a> { pub(crate) base: &'a str, pub(crate) paths: Option<&'a [String]>, @@ -104,6 +149,336 @@ pub(crate) fn open_project( }) } +pub(crate) fn context_packet( + view: &ProjectView, + options: ContextOptions<'_>, +) -> Result> { + let started = Instant::now(); + let budget = options.token_budget.clamp(128, 2_000); + let intent = if options.intent == ContextIntent::Auto { + classify_intent(options.query) + } else { + options.intent + }; + let cursor = options + .cursor + .map(decode_cursor) + .transpose()? + .unwrap_or_default(); + let query_hash = context_state::digest("context-query-v1", options.query); + if options.cursor.is_some() + && (cursor.query_hash != query_hash + || cursor.index_version != view.index.version + || cursor.intent != intent.as_str()) + { + return Err("cursor_stale: query, intent, or index changed; restart without cursor".into()); + } + + let selection = select_subgraph( + &view.index, + options.query, + SelectionOptions { max_nodes: 48 }, + ); + let ledger = LedgerStore::open(&view.root)?; + let projection = options + .source + .as_ref() + .map(|source| ledger.projection_for(source).text) + .unwrap_or_else(|| ledger.project_facts().text); + let sequence = ledger.events().len() as u64; + let mut records = Vec::new(); + if !projection.trim().is_empty() { + push_evidence( + &mut records, + 1_000, + format!("state {}", projection.replace('\n', "; ")), + ); + } + + let anchors = selection + .nodes + .iter() + .copied() + .filter(|node| view.index.graph.kind(*node) != "file") + .take(12) + .collect::>(); + let anchor_ids = anchors + .iter() + .enumerate() + .map(|(index, node)| (*node, format!("A{}", index + 1))) + .collect::>(); + for node in &anchors { + let id = &anchor_ids[node]; + let span = view.index.spans.get(node); + let path = node_path(view, *node).unwrap_or(""); + let qualified = view + .index + .qualified_names + .get(node) + .map(String::as_str) + .unwrap_or_else(|| view.index.graph.label(*node)); + push_evidence( + &mut records, + 900, + format!( + "{id} {} {qualified} @ {path}:{}-{}", + view.index.graph.kind(*node), + span.map(|span| span.start_line).unwrap_or(0), + span.map(|span| span.end_line).unwrap_or(0) + ), + ); + } + + if intent == ContextIntent::Change { + for line in changes( + view, + ChangeOptions { + base: "HEAD", + paths: None, + depth: 2, + include_tests: true, + token_budget: 2_000, + }, + ) + .lines() + .filter(|line| !line.trim().is_empty() && !line.starts_with('#')) + { + push_evidence(&mut records, 850, format!("change {line}")); + } + } + + if matches!( + intent, + ContextIntent::Flow | ContextIntent::Locate | ContextIntent::Inspect + ) { + for edge in &view.index.graph.edges { + let (Some(source), Some(target)) = + (anchor_ids.get(&edge.source), anchor_ids.get(&edge.target)) + else { + continue; + }; + let kind = view.index.graph.atom(edge.kind); + if kind != "contains" { + push_evidence(&mut records, 800, format!("E {source} {kind} {target}")); + } + } + } + + if intent == ContextIntent::Inspect { + for node in anchors.iter().take(3) { + let Some(path) = node_path(view, *node) else { + continue; + }; + let source = source_window(view, path, &[*node], 24, MAX_TEXT) + .unwrap_or_else(|error| format!("> Source unavailable: {error}")); + push_source_evidence( + &mut records, + 600, + format!("S {}\n{source}", anchor_ids[node]), + ); + } + } + if let Some(first) = anchors.first() { + push_evidence( + &mut records, + 100, + format!( + "next inspect {} file={}", + view.index.graph.label(*first), + node_path(view, *first).unwrap_or("") + ), + ); + } + + let envelope_reserve = packet_envelope_reserve( + budget, + intent, + view.index.version, + options.map_requested, + &query_hash, + records.len(), + ); + + let delivery = context_state::deliver( + &view.root, + records, + context_state::DeliveryRequest { + source: options.source.as_ref(), + requested: options.delivery, + token_budget: budget.saturating_sub(envelope_reserve).max(1), + offset: cursor.offset, + index_version: view.index.version, + ledger_sequence: sequence, + }, + ); + let packet_seed = delivery + .packed + .records + .iter() + .map(|record| record.id.as_str()) + .collect::>() + .join(":"); + let packet_id = &context_state::digest("context-packet-v1", &packet_seed)[..12]; + let mut lines = vec![String::new()]; + lines.extend( + delivery + .packed + .records + .iter() + .map(|record| record.text.clone()), + ); + if delivery.packed.records.is_empty() && delivery.duplicate_evidence > 0 { + lines.push(format!("unchanged sequence={sequence}")); + } + let next = delivery.packed.next_offset.map(|offset| { + encode_cursor(&ContextCursor { + query_hash: query_hash.clone(), + index_version: view.index.version, + intent: intent.as_str().into(), + offset, + }) + }); + lines.push(format!( + "omitted={}{}", + delivery.packed.omitted, + next.as_ref() + .map(|cursor| format!(" next={cursor}")) + .unwrap_or_default() + )); + let mut estimated_tokens = estimate_tokens(&lines.join("\n")); + let mut text = String::new(); + for _ in 0..4 { + lines[0] = format!( + "C1 id=p{packet_id} intent={} index=v{} budget={budget} used≈{estimated_tokens} delivery={} image_cost={}", + intent.as_str(), + view.index.version, + delivery.effective_delivery.as_str(), + if options.map_requested { + "provider" + } else { + "none" + } + ); + text = lines.join("\n"); + let rendered_estimate = estimate_tokens(&text); + if rendered_estimate == estimated_tokens { + break; + } + estimated_tokens = rendered_estimate; + } + context_state::record_metrics( + &view.root, + context_state::MetricSample { + intent: intent.as_str(), + estimated_tokens, + duplicates: delivery.duplicate_evidence, + map: options.map_requested, + error: false, + delivery: delivery.effective_delivery, + elapsed: started.elapsed(), + }, + ); + Ok(ContextPacket { text }) +} + +fn packet_envelope_reserve( + budget: usize, + intent: ContextIntent, + index_version: u32, + map_requested: bool, + query_hash: &str, + evidence_count: usize, +) -> usize { + let cursor = encode_cursor(&ContextCursor { + query_hash: query_hash.to_owned(), + index_version, + intent: intent.as_str().into(), + offset: evidence_count, + }); + let envelope = format!( + "C1 id=p000000000000 intent={} index=v{} budget={} used≈{} delivery=delta image_cost={}\nomitted={} next={}", + intent.as_str(), + index_version, + budget, + budget, + if map_requested { "provider" } else { "none" }, + evidence_count, + cursor, + ); + // Include slack for the two separators around evidence and digit-boundary + // changes when the final packet estimate is written into the header. + estimate_tokens(&envelope).saturating_add(2) +} + +fn classify_intent(query: &str) -> ContextIntent { + let query = query.to_ascii_lowercase(); + if [ + "resume", "continue", "pick up", "blocked", "failed", "failure", + ] + .iter() + .any(|term| query.contains(term)) + { + ContextIntent::Resume + } else if [ + "change", "changed", "impact", "worktree", "affected", "tests", + ] + .iter() + .any(|term| query.contains(term)) + { + ContextIntent::Change + } else if query.starts_with("how ") + || [" flow ", " reach ", " path ", " calls "] + .iter() + .any(|term| query.contains(term)) + { + ContextIntent::Flow + } else if ["source", "implementation", "inspect", "show code", "read "] + .iter() + .any(|term| query.contains(term)) + { + ContextIntent::Inspect + } else { + ContextIntent::Locate + } +} + +fn push_evidence(records: &mut Vec, priority: i32, text: String) { + records.push(EvidenceRecord { + id: context_state::digest("context-evidence-v1", &text), + priority, + text, + shrink_lines: false, + }); +} + +fn push_source_evidence(records: &mut Vec, priority: i32, text: String) { + records.push(EvidenceRecord { + id: context_state::digest("context-evidence-v1", &text), + priority, + text, + shrink_lines: true, + }); +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct ContextCursor { + query_hash: String, + index_version: u32, + intent: String, + offset: usize, +} + +fn encode_cursor(cursor: &ContextCursor) -> String { + URL_SAFE_NO_PAD.encode(serde_json::to_vec(cursor).unwrap_or_default()) +} + +fn decode_cursor(value: &str) -> Result> { + if value.len() > 2_048 { + return Err("cursor exceeds 2048 characters".into()); + } + Ok(serde_json::from_slice(&URL_SAFE_NO_PAD.decode(value)?)?) +} + pub(crate) fn search(view: &ProjectView, query: &str, kind: Option<&str>, limit: usize) -> String { let query = query.trim().to_ascii_lowercase(); let kind = kind.map(|kind| if kind == "type" { "type_alias" } else { kind }); @@ -1856,6 +2231,173 @@ mod tests { fs::remove_dir_all(root).unwrap(); } + #[test] + fn adaptive_context_is_budgeted_classified_and_session_deduplicated() { + let root = temp_root("adaptive-context-test"); + efficiency_fixture(&root); + let source = LedgerSource { + harness: "custom".into(), + session_id: "one".into(), + }; + LedgerStore::transaction(&root, |ledger| { + ledger.append_for( + source.clone(), + LedgerEventKind::EditObserved { + paths: vec!["app.py".into()], + }, + )?; + Ok(()) + }) + .unwrap(); + let autosync = AutoSync::default(); + let view = open_project(&autosync, root.to_str()).unwrap(); + let first = context_packet( + &view, + ContextOptions { + query: "How does run reach leaf?", + token_budget: 256, + intent: ContextIntent::Auto, + delivery: Delivery::Delta, + source: Some(source.clone()), + cursor: None, + map_requested: false, + }, + ) + .unwrap(); + assert!(first.text.contains("intent=flow")); + assert!(first.text.contains("image_cost=none")); + assert!(first.text.contains("state S")); + assert!(first.text.contains("A1")); + assert!(estimate_tokens(&first.text) <= 256); + let duplicate = context_packet( + &view, + ContextOptions { + query: "How does run reach leaf?", + token_budget: 256, + intent: ContextIntent::Auto, + delivery: Delivery::Delta, + source: Some(source.clone()), + cursor: None, + map_requested: false, + }, + ) + .unwrap(); + assert!(duplicate.text.contains("unchanged sequence=")); + assert!(!duplicate.text.contains("private source body")); + let full = context_packet( + &view, + ContextOptions { + query: "How does run reach leaf?", + token_budget: 256, + intent: ContextIntent::Flow, + delivery: Delivery::Full, + source: Some(source), + cursor: None, + map_requested: false, + }, + ) + .unwrap(); + assert!(full.text.contains("delivery=full")); + assert!(full.text.contains("A1")); + drop(view); + drop(autosync); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn rendered_context_packets_respect_public_budget_boundaries() { + let root = temp_root("context-budget-boundaries-test"); + efficiency_fixture(&root); + let autosync = AutoSync::default(); + let view = open_project(&autosync, root.to_str()).unwrap(); + for budget in [128, 600, 2_000] { + for map_requested in [false, true] { + let packet = context_packet( + &view, + ContextOptions { + query: "inspect implementation run helper leaf", + token_budget: budget, + intent: ContextIntent::Inspect, + delivery: Delivery::Delta, + source: Some(LedgerSource { + harness: "budget-test".into(), + session_id: format!("{budget}-{map_requested}"), + }), + cursor: None, + map_requested, + }, + ) + .unwrap(); + let declared = packet + .text + .lines() + .next() + .and_then(|header| { + header + .split_whitespace() + .find_map(|field| field.strip_prefix("used≈")) + }) + .and_then(|used| used.parse::().ok()) + .unwrap(); + assert!( + estimate_tokens(&packet.text) <= budget, + "{} token packet exceeded its {} token budget:\n{}", + estimate_tokens(&packet.text), + budget, + packet.text + ); + assert_eq!(declared, estimate_tokens(&packet.text)); + } + } + drop(view); + drop(autosync); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn context_cursor_rejects_a_changed_query() { + let root = temp_root("context-cursor-test"); + efficiency_fixture(&root); + let autosync = AutoSync::default(); + let view = open_project(&autosync, root.to_str()).unwrap(); + let first = context_packet( + &view, + ContextOptions { + query: "inspect implementation run helper leaf", + token_budget: 128, + intent: ContextIntent::Inspect, + delivery: Delivery::Full, + source: None, + cursor: None, + map_requested: false, + }, + ) + .unwrap(); + let cursor = first + .text + .split("next=") + .nth(1) + .expect("bounded packet has a continuation") + .trim(); + let error = context_packet( + &view, + ContextOptions { + query: "different query", + token_budget: 128, + intent: ContextIntent::Inspect, + delivery: Delivery::Full, + source: None, + cursor: Some(cursor), + map_requested: false, + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("cursor_stale")); + drop(view); + drop(autosync); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn frozen_efficiency_scenarios_reduce_median_tool_calls_by_forty_percent() { let fixture: serde_json::Value = serde_json::from_str(include_str!( diff --git a/crates/spectra-cli/tests/cli.rs b/crates/spectra-cli/tests/cli.rs index 270db9a..987bf48 100644 --- a/crates/spectra-cli/tests/cli.rs +++ b/crates/spectra-cli/tests/cli.rs @@ -80,6 +80,86 @@ fn init_and_map_complete_end_to_end() { fs::remove_dir_all(root).unwrap(); } +#[test] +fn adaptive_context_cli_deduplicates_tracks_metrics_and_maps_only_explicitly() { + let root = fixture(); + let binary = env!("CARGO_BIN_EXE_spectra"); + let run_context = |representation: &str| { + Command::new(binary) + .args([ + "context", + "How does entry reach worker?", + "--path", + root.to_str().unwrap(), + "--token-budget", + "256", + "--source-harness", + "custom", + "--session-id", + "private-session-id", + "--representation", + representation, + ]) + .output() + .unwrap() + }; + let first = run_context("text"); + assert!( + first.status.success(), + "{}", + String::from_utf8_lossy(&first.stderr) + ); + let first = String::from_utf8(first.stdout).unwrap(); + assert!(first.starts_with("C1 id=p")); + assert!(first.contains("intent=flow")); + assert!(first.contains("image_cost=none")); + assert!(!root.join(".spectra/artifacts").exists()); + + let duplicate = run_context("text"); + assert!(duplicate.status.success()); + assert!( + String::from_utf8(duplicate.stdout) + .unwrap() + .contains("unchanged sequence=") + ); + let receipts = fs::read_to_string(root.join(".spectra/context-receipts-v1.json")).unwrap(); + assert!(!receipts.contains("private-session-id")); + assert!(!receipts.contains("pub fn entry")); + + let mapped = run_context("map"); + assert!( + mapped.status.success(), + "{}", + String::from_utf8_lossy(&mapped.stderr) + ); + let mapped = String::from_utf8(mapped.stdout).unwrap(); + assert!(mapped.contains("image_cost=provider")); + assert!(mapped.contains("PNG ")); + + let stats = Command::new(binary) + .args(["stats", "--path", root.to_str().unwrap(), "--json"]) + .output() + .unwrap(); + let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap(); + assert_eq!(stats["calls"], 3); + assert!(stats["duplicate_evidence_avoided"].as_u64().unwrap() > 0); + assert_eq!(stats["maps_requested"], 1); + + let disabled = Command::new(binary) + .env("SPECTRA_METRICS", "off") + .args(["context", "find entry", "--path", root.to_str().unwrap()]) + .output() + .unwrap(); + assert!(disabled.status.success()); + let stats = Command::new(binary) + .args(["stats", "--path", root.to_str().unwrap(), "--json"]) + .output() + .unwrap(); + let stats: serde_json::Value = serde_json::from_slice(&stats.stdout).unwrap(); + assert_eq!(stats["calls"], 3); + fs::remove_dir_all(root).unwrap(); +} + #[test] fn mcp_handshake_does_not_index_before_the_first_tool_call() { let root = fixture(); diff --git a/crates/spectra-core/src/context.rs b/crates/spectra-core/src/context.rs new file mode 100644 index 0000000..1ba9847 --- /dev/null +++ b/crates/spectra-core/src/context.rs @@ -0,0 +1,192 @@ +//! Model-facing context budgeting primitives. + +/// A complete unit of model-facing evidence. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EvidenceRecord { + pub id: String, + pub priority: i32, + pub text: String, + /// Source windows may discard complete trailing lines to fit a packet. + /// Anchors, relations, and every other record remain indivisible. + pub shrink_lines: bool, +} + +/// The result of packing evidence into a bounded context packet. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct PackedEvidence { + pub records: Vec, + pub estimated_tokens: usize, + pub omitted: usize, + pub next_offset: Option, +} + +/// Conservatively estimates tokens without depending on a provider tokenizer. +/// +/// Code and paths contain more punctuation than prose, so bytes-per-token alone +/// is not sufficient. The lexical estimate counts transitions into runs of +/// alphanumeric or punctuation characters, then keeps the larger estimate and +/// reserves five percent for provider variance. +pub fn estimate_tokens(text: &str) -> usize { + if text.is_empty() { + return 0; + } + let bytes = text.len().div_ceil(3); + let mut segments = 0_usize; + let mut previous = CharacterClass::Whitespace; + for ch in text.chars() { + let class = if ch.is_whitespace() { + CharacterClass::Whitespace + } else if ch.is_alphanumeric() || ch == '_' { + CharacterClass::Lexical + } else { + CharacterClass::Punctuation + }; + if class != CharacterClass::Whitespace && class != previous { + segments += 1; + } + previous = class; + } + bytes.max(segments).saturating_mul(105).div_ceil(100) +} + +/// Packs complete evidence records by priority and stable input order. +pub fn pack_evidence( + records: impl IntoIterator, + token_budget: usize, + offset: usize, +) -> PackedEvidence { + let budget = token_budget.max(1); + let mut ranked = records.into_iter().enumerate().collect::>(); + ranked.sort_by_key(|(index, record)| (std::cmp::Reverse(record.priority), *index)); + let total = ranked.len(); + let mut packed = PackedEvidence::default(); + let mut next = None; + for (rank, (_, mut record)) in ranked.into_iter().enumerate().skip(offset) { + let remaining = budget.saturating_sub(packed.estimated_tokens); + if record.shrink_lines { + shrink_source_window(&mut record.text, remaining); + } + let candidate = packed + .records + .iter() + .map(|record| record.text.as_str()) + .chain(std::iter::once(record.text.as_str())) + .collect::>() + .join("\n"); + let tokens = estimate_tokens(&candidate); + if tokens <= budget { + packed.estimated_tokens = tokens; + packed.records.push(record); + } else { + next = Some(rank); + packed.omitted += total.saturating_sub(rank); + break; + } + } + packed.omitted += offset.min(total); + packed.next_offset = next; + packed +} + +fn shrink_source_window(text: &mut String, token_budget: usize) { + if estimate_tokens(text) <= token_budget { + return; + } + let mut lines = text.lines().collect::>(); + while lines.len() > 2 && estimate_tokens(&lines.join("\n")) > token_budget { + lines.pop(); + } + *text = lines.join("\n"); +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum CharacterClass { + Whitespace, + Lexical, + Punctuation, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn estimator_is_conservative_for_code_and_unicode() { + assert!(estimate_tokens("fn render_map(a: &str) -> Result<()> {") >= 12); + assert!(estimate_tokens("🔐 secret") >= 4); + assert_eq!(estimate_tokens(""), 0); + } + + #[test] + fn packer_keeps_records_atomic_and_stable() { + let records = [ + EvidenceRecord { + id: "low".into(), + priority: 1, + text: "low priority evidence".into(), + shrink_lines: false, + }, + EvidenceRecord { + id: "first".into(), + priority: 10, + text: "first anchor".into(), + shrink_lines: false, + }, + EvidenceRecord { + id: "second".into(), + priority: 10, + text: "second anchor".into(), + shrink_lines: false, + }, + ]; + let budget = estimate_tokens("first anchor") + estimate_tokens("second anchor"); + let packed = pack_evidence(records, budget, 0); + assert_eq!( + packed + .records + .iter() + .map(|record| record.id.as_str()) + .collect::>(), + ["first", "second"] + ); + assert_eq!(packed.omitted, 1); + assert_eq!(packed.next_offset, Some(2)); + } + + #[test] + fn packer_shrinks_only_source_windows_by_complete_lines() { + let record = EvidenceRecord { + id: "source".into(), + priority: 1, + text: "S A1\n10\tfirst source line\n11\tsecond source line\n12\tthird source line" + .into(), + shrink_lines: true, + }; + let budget = estimate_tokens("S A1\n10\tfirst source line"); + let packed = pack_evidence([record], budget, 0); + assert_eq!(packed.records.len(), 1); + assert_eq!(packed.records[0].text, "S A1\n10\tfirst source line"); + } + + #[test] + fn packer_accounts_for_separators_between_records() { + let text = "x".repeat(300); + let records = (0..2).map(|index| EvidenceRecord { + id: index.to_string(), + priority: 1, + text: text.clone(), + shrink_lines: false, + }); + let budget = estimate_tokens(&text) * 2; + let packed = pack_evidence(records, budget, 0); + let joined = packed + .records + .iter() + .map(|record| record.text.as_str()) + .collect::>() + .join("\n"); + assert_eq!(packed.records.len(), 1); + assert_eq!(packed.omitted, 1); + assert!(estimate_tokens(&joined) <= budget); + } +} diff --git a/crates/spectra-core/src/lib.rs b/crates/spectra-core/src/lib.rs index 30c2318..06bf0ca 100644 --- a/crates/spectra-core/src/lib.rs +++ b/crates/spectra-core/src/lib.rs @@ -3,6 +3,7 @@ use std::path::Path; mod adapters; +pub mod context; mod error; pub mod graph; mod index; @@ -10,6 +11,7 @@ pub mod ledger; mod select; pub use adapters::{SupportedLanguage, is_supported_path, supported_languages}; +pub use context::{EvidenceRecord, PackedEvidence, estimate_tokens, pack_evidence}; pub use error::{Error, Result}; pub use index::{CodeIndex, INDEX_VERSION, IndexReport, SourceSpan}; pub use ledger::{ diff --git a/crates/spectra-core/src/select.rs b/crates/spectra-core/src/select.rs index 08e8a5b..5cdfe84 100644 --- a/crates/spectra-core/src/select.rs +++ b/crates/spectra-core/src/select.rs @@ -115,18 +115,8 @@ pub fn select_subgraph(index: &CodeIndex, query: &str, options: SelectionOptions // become direct anchors. They otherwise crowd out actual symbols // when a repository or crate name appears in the question. if group_score >= 35 { - let context_bonus = - contextual_path_bonus(&term_groups[group_index], &label, &path); - let architecture_bonus = - architecture_path_bonus(&term_groups[group_index], &flattened_terms, &path); let file_affinity = symbol_file_affinity_bonus(&label, &path); - let api_boundary_bonus = if matches!(kind, "function" | "method") - && (path.ends_with("/lib.rs") || path.ends_with("/mod.rs")) - { - 100 - } else { - 0 - }; + let api_boundary_bonus = public_boundary_bonus(kind, &path); let compound_label_bonus = label_words .iter() .filter(|word| term_groups[group_index].contains(word)) @@ -141,8 +131,6 @@ pub fn select_subgraph(index: &CodeIndex, query: &str, options: SelectionOptions scored_by_group[group_index].push(( Reverse( group_score - + context_bonus - + architecture_bonus + file_affinity + api_boundary_bonus + compound_label_bonus @@ -361,131 +349,22 @@ fn add_relationship_connectors( ); } -fn contextual_path_bonus(group: &[String], label: &str, path: &str) -> i32 { - let cli_concept = group - .iter() - .any(|term| matches!(term.as_str(), "arg" | "args" | "cli")); - let config_concept = group - .iter() - .any(|term| matches!(term.as_str(), "config" | "configure")); - let transition_concept = group.iter().any(|term| term == "from_low_args"); - if !(cli_concept || config_concept || transition_concept) { +fn public_boundary_bonus(kind: &str, path: &str) -> i32 { + if !matches!(kind, "function" | "method" | "route" | "component") { return 0; } - let mut bonus = 0; - if path.contains("/flags/") || path.starts_with("flags/") { - bonus += if cli_concept { 120 } else { 80 }; - } else if path.contains("/cli/") || path.starts_with("cli/") { - bonus += if cli_concept { 50 } else { 20 }; - } - if cli_concept && label.contains("parse") { - bonus += 70; - } - if transition_concept && label.contains("parse") { - bonus += 140; - } - bonus -} - -fn architecture_path_bonus(group: &[String], terms: &BTreeSet<&str>, path: &str) -> i32 { - let scheduler_context = terms - .iter() - .any(|term| matches!(*term, "schedule" | "scheduler" | "worker")); - let scheduler_group = group.iter().any(|term| { - matches!( - term.as_str(), - "execute" | "poll" | "run" | "schedule" | "scheduler" | "worker" - ) - }); - let lsp_dispatch_context = terms.iter().any(|term| *term == "lsp") - && terms - .iter() - .any(|term| matches!(*term, "dispatch" | "dispatcher" | "handler")); - let timer_context = terms - .iter() - .any(|term| matches!(*term, "sleep" | "timer" | "deadline")) - && terms.iter().any(|term| *term == "driver"); - let timer_group = group.iter().any(|term| { - matches!( - term.as_str(), - "deadline" - | "driver" - | "elapsed" - | "expiration" - | "poll" - | "ready" - | "time" - | "timer" - | "wake" - ) - }); - let timer_entry_group = group.iter().any(|term| { - matches!( - term.as_str(), - "deadline" | "elapsed" | "expiration" | "poll" | "ready" | "wake" - ) - }); - let sleep_group = group.iter().any(|term| term == "sleep"); - let completion_context = terms.iter().any(|term| *term == "completion") - && terms - .iter() - .any(|term| matches!(*term, "analysis" | "context" | "collect")); - let completion_group = group.iter().any(|term| { - matches!( - term.as_str(), - "analysis" | "collect" | "completion" | "context" | "item" | "items" - ) - }); - let search_pipeline_context = terms.iter().any(|term| *term == "search") - && terms - .iter() - .any(|term| matches!(*term, "haystack" | "walk" | "worker")); - let search_pipeline_group = group.iter().any(|term| { - matches!( - term.as_str(), - "execute" | "haystack" | "path" | "search" | "walk" | "worker" - ) - }); - let mut bonus = 0; - if scheduler_context && scheduler_group { - if path.contains("/runtime/") { - bonus += 100; - } - if path.contains("/scheduler/") { - bonus += 50; - } - } - if lsp_dispatch_context && path.contains("/handlers/") { - bonus += 80; - } - if timer_context && timer_group && path.contains("/runtime/time/") { - bonus += 120; - } - if timer_context && timer_entry_group && path.ends_with("/runtime/time/entry.rs") { - bonus += 180; - } - if timer_context && sleep_group && path.contains("/time/") { - bonus += 80; - } - if completion_context && completion_group { - if path.contains("/ide-completion/") { - bonus += 120; - } else if path.contains("/ide/src/") { - bonus += 80; - } - } - if search_pipeline_context && search_pipeline_group { - if path.ends_with("/core/search.rs") { - bonus += 220; - } else if path.ends_with("/core/main.rs") { - bonus += 180; - } else if path.ends_with("/core/haystack.rs") { - bonus += 120; - } else if path.ends_with("/ignore/src/walk.rs") { - bonus += 80; - } + let stem = path + .rsplit('/') + .next() + .and_then(|file| file.split('.').next()) + .unwrap_or(""); + if matches!(stem, "lib" | "mod" | "index" | "api" | "router" | "routes") { + 100 + } else if path.contains("/api/") || path.contains("/routes/") { + 60 + } else { + 0 } - bonus } fn symbol_file_affinity_bonus(label: &str, path: &str) -> i32 { @@ -580,23 +459,6 @@ fn terms(query: &str) -> Vec> { .replace("command line", "cli") .replace("high-level", "") .replace("high level", ""); - let cli_to_config = (normalized.contains("cli") || normalized.contains("argument")) - && (normalized.contains("config") || normalized.contains("configure")); - if cli_to_config { - groups.push( - [ - "convert", - "conversion", - "from_low_args", - "hiargs", - "lowargs", - "parse", - ] - .into_iter() - .map(str::to_owned) - .collect(), - ); - } for raw in normalized.split(|ch: char| !ch.is_alphanumeric() && ch != '_') { let value = raw.trim_matches('_').to_ascii_lowercase(); if value.len() > 1 && !is_stop_word(&value) { @@ -643,16 +505,10 @@ fn term_variants(value: &str) -> Vec { "command" => &["cli"], "configuration" | "configured" => &["config", "configure"], "directory" | "directories" => &["dir"], - "deadline" => &["elapsed", "expiration"], "dispatch" | "dispatched" => &["dispatcher", "route"], - "driver" => &["park", "process"], - "poll" | "polled" => &["execute", "run"], - "ready" => &["elapsed", "poll", "wake"], - "readiness" => &["ready"], "register" | "registered" => &["registration"], "scheduler" => &["schedule"], "spawn" | "spawned" => &["spawner"], - "timer" => &["time"], _ => &[], }; variants.extend(aliases.iter().map(|alias| (*alias).to_owned())); @@ -784,7 +640,7 @@ mod tests { assert!(term_variants("arguments").contains(&"args".to_owned())); assert!(term_variants("parsed").contains(&"parse".to_owned())); assert!(term_variants("dispatch").contains(&"dispatcher".to_owned())); - assert!(term_variants("polled").contains(&"run".to_owned())); + assert!(term_variants("polled").contains(&"poll".to_owned())); } #[test] @@ -846,6 +702,26 @@ mod tests { ); } + #[test] + fn production_selector_contains_no_frozen_corpus_rules() { + let source = include_str!("select.rs") + .split("#[cfg(test)]") + .next() + .unwrap_or_default(); + for forbidden in [ + ["from", "low", "args"].join("_"), + ["hi", "args"].join(""), + ["hay", "stack"].join(""), + ["ide", "completion"].join("-"), + ["runtime", "time", "entry.rs"].join("/"), + ] { + assert!( + !source.contains(&forbidden), + "frozen selector rule: {forbidden}" + ); + } + } + #[test] fn adds_callers_that_bridge_matched_symbols() { let mut graph = PackedGraph::default(); diff --git a/crates/spectra-render/Cargo.toml b/crates/spectra-render/Cargo.toml index e85882d..48825f6 100644 --- a/crates/spectra-render/Cargo.toml +++ b/crates/spectra-render/Cargo.toml @@ -17,5 +17,5 @@ name = "spectra_render" [dependencies] resvg.workspace = true -spectra-core = { package = "spectra-context-core", version = "0.3.0", path = "../spectra-core" } +spectra-core = { package = "spectra-context-core", version = "0.4.0", path = "../spectra-core" } tiny-skia.workspace = true diff --git a/docs/state-machine-ledger.md b/docs/state-machine-ledger.md index 7d3424f..1868ac8 100644 --- a/docs/state-machine-ledger.md +++ b/docs/state-machine-ledger.md @@ -41,7 +41,7 @@ Raw event-history exposure is intentionally not part of the agent interface. ## Automatic operation -The Ledger is lazily created through normal Spectra use. `spectra brief`, `spectra map`, and their MCP equivalents synchronize the repository before selection and reuse one opened index snapshot for the complete response. Brief reads project-wide Ledger facts without borrowing another harness session's state; callers must supply exact source metadata to receive session state. Users do not need an initialization command, daemon, or manual synchronization step. +The Ledger is lazily created through normal Spectra use. `spectra context`, `spectra brief`, `spectra map`, and their MCP equivalents synchronize the repository before selection and reuse one opened index snapshot for the complete response. Context and brief read project-wide Ledger facts without borrowing another harness session's state; callers must supply exact source metadata to receive session state. Context delivery receipts are mutable runtime state stored separately from this immutable Ledger. Users do not need an initialization command, daemon, or manual synchronization step. Verified provider adapters translate documented lifecycle events into the same Ledger facts: diff --git a/install.ps1 b/install.ps1 new file mode 100644 index 0000000..612d5f0 --- /dev/null +++ b/install.ps1 @@ -0,0 +1,46 @@ +$ErrorActionPreference = "Stop" + +$Repository = "https://github.com/rankupgames/Spectra" +$Version = if ($env:SPECTRA_VERSION) { $env:SPECTRA_VERSION } else { "latest" } +$InstallDir = if ($env:SPECTRA_INSTALL_DIR) { + $env:SPECTRA_INSTALL_DIR +} else { + Join-Path $env:LOCALAPPDATA "Programs\Spectra" +} + +if (-not [Environment]::Is64BitOperatingSystem) { + throw "Spectra requires 64-bit Windows" +} + +if ($Version -eq "latest") { + $ReleaseUrl = "$Repository/releases/latest/download" +} else { + $Tag = if ($Version.StartsWith("v")) { $Version } else { "v$Version" } + $ReleaseUrl = "$Repository/releases/download/$Tag" +} + +$Archive = "spectra-x86_64-pc-windows-msvc.zip" +$Temporary = Join-Path ([IO.Path]::GetTempPath()) ("spectra-install-" + [Guid]::NewGuid()) +New-Item -ItemType Directory -Path $Temporary | Out-Null +try { + Invoke-WebRequest "$ReleaseUrl/$Archive" -OutFile (Join-Path $Temporary $Archive) + Invoke-WebRequest "$ReleaseUrl/SHA256SUMS" -OutFile (Join-Path $Temporary "SHA256SUMS") + $ChecksumLine = Get-Content (Join-Path $Temporary "SHA256SUMS") | + Where-Object { $_ -match "\s$([regex]::Escape($Archive))$" } | + Select-Object -First 1 + if (-not $ChecksumLine) { + throw "Release checksum is missing $Archive" + } + $Expected = ($ChecksumLine -split "\s+")[0].ToLowerInvariant() + $Actual = (Get-FileHash (Join-Path $Temporary $Archive) -Algorithm SHA256).Hash.ToLowerInvariant() + if ($Actual -ne $Expected) { + throw "Checksum verification failed" + } + Expand-Archive (Join-Path $Temporary $Archive) -DestinationPath $Temporary -Force + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + Copy-Item (Join-Path $Temporary "spectra.exe") (Join-Path $InstallDir "spectra.exe") -Force + Write-Host "Installed Spectra to $InstallDir\spectra.exe" + Write-Host "Add $InstallDir to PATH, then run: spectra install" +} finally { + Remove-Item -Recurse -Force $Temporary -ErrorAction SilentlyContinue +} diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..85ac35e --- /dev/null +++ b/install.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +repository=https://github.com/rankupgames/Spectra +version=${SPECTRA_VERSION:-latest} +install_dir=${SPECTRA_INSTALL_DIR:-"$HOME/.local/bin"} + +case "$(uname -s)" in + Darwin) os=apple-darwin ;; + Linux) os=unknown-linux-gnu ;; + *) echo "spectra: unsupported operating system" >&2; exit 1 ;; +esac + +case "$(uname -m)" in + arm64|aarch64) arch=aarch64 ;; + x86_64|amd64) arch=x86_64 ;; + *) echo "spectra: unsupported CPU architecture" >&2; exit 1 ;; +esac + +if [ "$version" = latest ]; then + release_url="$repository/releases/latest/download" +else + case "$version" in v*) tag=$version ;; *) tag=v$version ;; esac + release_url="$repository/releases/download/$tag" +fi + +archive="spectra-$arch-$os.tar.gz" +temporary=$(mktemp -d "${TMPDIR:-/tmp}/spectra-install.XXXXXX") +trap 'rm -rf "$temporary"' EXIT INT TERM + +curl --fail --location --silent --show-error "$release_url/$archive" -o "$temporary/$archive" +curl --fail --location --silent --show-error "$release_url/SHA256SUMS" -o "$temporary/SHA256SUMS" +expected=$(awk -v file="$archive" '$2 == file { print $1 }' "$temporary/SHA256SUMS") +if [ -z "$expected" ]; then + echo "spectra: release checksum is missing $archive" >&2 + exit 1 +fi +actual=$(shasum -a 256 "$temporary/$archive" | awk '{ print $1 }') +if [ "$actual" != "$expected" ]; then + echo "spectra: checksum verification failed" >&2 + exit 1 +fi + +tar -xzf "$temporary/$archive" -C "$temporary" +mkdir -p "$install_dir" +install -m 0755 "$temporary/spectra" "$install_dir/spectra" +echo "Installed Spectra to $install_dir/spectra" +case ":${PATH}:" in + *":$install_dir:"*) ;; + *) echo "Add $install_dir to PATH, then run: spectra install" ;; +esac diff --git a/packaging/homebrew/spectra.rb.template b/packaging/homebrew/spectra.rb.template new file mode 100644 index 0000000..fdfa8d6 --- /dev/null +++ b/packaging/homebrew/spectra.rb.template @@ -0,0 +1,34 @@ +class Spectra < Formula + desc "Token-efficient code context runtime for local AI agents" + homepage "https://github.com/rankupgames/Spectra" + version "@VERSION@" + license "MIT" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/rankupgames/Spectra/releases/download/@TAG@/spectra-aarch64-apple-darwin.tar.gz" + sha256 "@MACOS_ARM64_SHA@" + else + url "https://github.com/rankupgames/Spectra/releases/download/@TAG@/spectra-x86_64-apple-darwin.tar.gz" + sha256 "@MACOS_X64_SHA@" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/rankupgames/Spectra/releases/download/@TAG@/spectra-aarch64-unknown-linux-gnu.tar.gz" + sha256 "@LINUX_ARM64_SHA@" + else + url "https://github.com/rankupgames/Spectra/releases/download/@TAG@/spectra-x86_64-unknown-linux-gnu.tar.gz" + sha256 "@LINUX_X64_SHA@" + end + end + + def install + bin.install "spectra" + end + + test do + assert_match version.to_s, shell_output("#{bin}/spectra --version") + end +end diff --git a/packaging/render-manifests.sh b/packaging/render-manifests.sh new file mode 100755 index 0000000..b76a440 --- /dev/null +++ b/packaging/render-manifests.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu + +tag=${1:?release tag is required} +checksums=${2:?checksum file is required} +output=${3:?output directory is required} +version=${tag#v} + +case "$tag" in + v[0-9]*) ;; + *) echo "spectra: release tag must be v-prefixed" >&2; exit 1 ;; +esac + +checksum() { + value=$(awk -v file="$1" '$2 == file { print $1 }' "$checksums") + if [ -z "$value" ]; then + echo "spectra: checksum is missing $1" >&2 + exit 1 + fi + printf '%s' "$value" +} + +render() { + template=$1 + destination=$2 + sed \ + -e "s|@TAG@|$tag|g" \ + -e "s|@VERSION@|$version|g" \ + -e "s|@MACOS_ARM64_SHA@|$(checksum spectra-aarch64-apple-darwin.tar.gz)|g" \ + -e "s|@MACOS_X64_SHA@|$(checksum spectra-x86_64-apple-darwin.tar.gz)|g" \ + -e "s|@LINUX_ARM64_SHA@|$(checksum spectra-aarch64-unknown-linux-gnu.tar.gz)|g" \ + -e "s|@LINUX_X64_SHA@|$(checksum spectra-x86_64-unknown-linux-gnu.tar.gz)|g" \ + -e "s|@WINDOWS_X64_SHA@|$(checksum spectra-x86_64-pc-windows-msvc.zip)|g" \ + "$template" > "$destination" +} + +mkdir -p "$output" +render packaging/homebrew/spectra.rb.template "$output/spectra.rb" +render packaging/scoop/spectra.json.template "$output/spectra.json" diff --git a/packaging/scoop/spectra.json.template b/packaging/scoop/spectra.json.template new file mode 100644 index 0000000..283d618 --- /dev/null +++ b/packaging/scoop/spectra.json.template @@ -0,0 +1,21 @@ +{ + "version": "@VERSION@", + "description": "Token-efficient code context runtime for local AI agents", + "homepage": "https://github.com/rankupgames/Spectra", + "license": "MIT", + "architecture": { + "64bit": { + "url": "https://github.com/rankupgames/Spectra/releases/download/@TAG@/spectra-x86_64-pc-windows-msvc.zip", + "hash": "@WINDOWS_X64_SHA@" + } + }, + "bin": "spectra.exe", + "checkver": "github", + "autoupdate": { + "architecture": { + "64bit": { + "url": "https://github.com/rankupgames/Spectra/releases/download/v$version/spectra-x86_64-pc-windows-msvc.zip" + } + } + } +}