diff --git a/.githooks/README.md b/.githooks/README.md index ed16d300..7081cc5d 100644 --- a/.githooks/README.md +++ b/.githooks/README.md @@ -12,12 +12,36 @@ never seems to fire, check that setting first. | Hook | What it does | |---|---| -| `pre-commit` | Runs `cargo fmt` and stages the result, so CI's fmt check can't fail. | +| `pre-commit` | Runs `cargo fmt` and stages the result, so CI's fmt check can't fail. Then runs the branch-aware root-md allowlist guard (see below) — on `develop`/`master`, `AGENTS.md`/`CLAUDE.md` are not allowlisted, so (re)introducing them via a direct commit or conflict resolution is blocked. | +| `pre-merge-commit` | Runs the same root-md allowlist guard on the merge result — a clean merge never invokes `pre-commit`, so this is the gate that stops `AGENTS.md`/`CLAUDE.md` riding a clean feature→`develop` merge onto the integration branch. | | `pre-push` | Blocks direct pushes to `master`; runs the QC gate (skipped when the branch changes no Rust); scans tracked files for customer references. | -| `post-checkout` | Creates `AGENTS.md` from `AGENTS.develop.md` on branch switch, if absent. | +| `post-checkout` | Branch-local agent-file lifecycle: on a feature branch, creates `AGENTS.md` from `AGENTS.develop.md` plus the `CLAUDE.md` pointer when absent; on `develop`/`master`, removes untracked `AGENTS.md`/`CLAUDE.md` leftovers. No-op on detached HEAD. | Any hook can be bypassed with `git push --no-verify` / `git commit --no-verify`. +## Root-md allowlist guard (`pre-commit`) + +Agents love dropping `*.md` files (diagnoses, plans, worklogs, test scenarios) +at the repo root. The `pre-commit` hook rejects any commit that **introduces** +(added/copied/renamed) a root-level `*.md` outside this allowlist: + +- `AGENTS.develop.md` — agent instructions reference (always allowed) +- `AGENTS.md`, `CLAUDE.md` — branch-local agent working files: allowlisted on + feature branches only. They are dropped at feature finalization; on + `develop`/`master` introducing them is rejected, merge commits included. +- `README.md`, `README_CSharp.md` — user-facing docs +- `CHANGELOG.md`, `RELEASING.md` — release infrastructure + +Everything else belongs in **`.docs/`** (gitignored, local-only) — see +AGENTS.develop.md, section *Root file hygiene (markdown)*. The guard itself +lives in [`lib/root-md-guard.sh`](lib/root-md-guard.sh), shared by +`pre-commit` and `pre-merge-commit`; its behavior is pinned by +`scripts/test-githooks.sh`. Only introductions are +checked: modifying an already-tracked stray is only possible after a +deliberate `--no-verify` bypass, where the file itself — not the commit — is +the violation. Dot-folders (`.githooks/`, `.github/`, `.claude/`, …) are out +of scope: a root-level file cannot be inside one. + ## `customer-patterns.local` The `pre-push` leak scan reads its patterns from `.githooks/customer-patterns.local` diff --git a/.githooks/lib/root-md-guard.sh b/.githooks/lib/root-md-guard.sh new file mode 100644 index 00000000..d22878a1 --- /dev/null +++ b/.githooks/lib/root-md-guard.sh @@ -0,0 +1,67 @@ +# root-md-guard.sh — shared root-md allowlist guard (sourced, never executed). +# Used by pre-commit (direct commits, conflict resolutions) and +# pre-merge-commit (clean merges never invoke pre-commit). +# +# The repo root keeps only its sanctioned markdown files (AGENTS.develop.md, +# section "Root file hygiene (markdown)"). Anything else — diagnoses, plans, +# test scenarios, worklogs — belongs in .docs/ (gitignored, local-only). +# +# The allowlist is branch-aware: AGENTS.md and CLAUDE.md are branch-local +# working files, dropped at feature finalization — on develop/master they are +# NOT allowlisted, so introducing them there is rejected, clean merge or not. +# +# Only introductions are checked (added/copied/renamed). Modifying an already +# tracked stray is only possible after a deliberate --no-verify bypass — the +# file itself is the violation then, and review catches it. Dot-folders +# (.githooks/, .github/, .claude/, ...) are out of scope by construction: a +# root-level file cannot be inside one. +# +# bash (not sh): process substitution. macOS bash 3.2 compatible. + +root_md_guard() { + # git branch --show-current: empty on detached HEAD (unborn HEAD prints + # the branch name — harmless feature-branch treatment). rev-parse + # --abbrev-ref would print "HEAD" when detached — creating branch-local + # files there would later block rebase/bisect checkouts of commits that + # track them — and exits 128 on unborn HEAD, killing a set -e caller + # mid-hook. + BRANCH=$(git branch --show-current 2>/dev/null || true) + case "$BRANCH" in + develop|master) + ALLOWED_ROOT_MD="AGENTS.develop.md README.md README_CSharp.md CHANGELOG.md RELEASING.md" + ;; + *) + ALLOWED_ROOT_MD="AGENTS.md AGENTS.develop.md CLAUDE.md README.md README_CSharp.md CHANGELOG.md RELEASING.md" + ;; + esac + + while IFS= read -r staged; do + # root level = no slash anywhere in the staged (post-rename) path + case "$staged" in + */*) continue ;; + esac + # tr, not ${var,,}: the parameter-expansion lowercase needs bash >= 4 and + # stock macOS still ships bash 3.2, where it is a fatal "bad substitution" + # that would block EVERY commit containing a root-level file. + case "$(printf '%s' "$staged" | tr 'A-Z' 'a-z')" in + *.md) ;; + *) continue ;; + esac + for allowed in $ALLOWED_ROOT_MD; do + [ "$staged" = "$allowed" ] && continue 2 + done + echo "" + echo "guard: BLOCKED — root-level '$staged' is not on the md allowlist for branch '${BRANCH:-detached}'." + echo " Root markdown is limited to: $ALLOWED_ROOT_MD" + echo " AGENTS.md/CLAUDE.md are branch-local: feature branches only, dropped at finalization." + echo " Diagnoses, plans, worklogs and test scenarios belong in .docs/ (gitignored)." + echo " See AGENTS.develop.md, section \"Root file hygiene (markdown)\"." + echo " Deliberate? Use: git commit --no-verify / git merge --no-verify" + echo "" + return 1 +# core.quotePath=false: with the default (true), git C-quotes non-ASCII paths +# (DIAGNOSE_ü.md -> "DIAGNOSE_\303\274.md"), and the quoted trailing `"` makes +# the *.md pattern miss — a stray with a non-ASCII name would sail through. + done < <(git -c core.quotePath=false diff --cached --name-only --diff-filter=ACR) + return 0 +} diff --git a/.githooks/post-checkout b/.githooks/post-checkout index f049da2e..6b5658fb 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -1,16 +1,51 @@ #!/bin/sh -# post-checkout hook — auto-create AGENTS.md from AGENTS.develop.md on branch switch. +# post-checkout hook — branch-local AGENTS.md/CLAUDE.md lifecycle on branch switch. # Runs after: git checkout, git switch, git checkout -b # -# Only copies if AGENTS.md does not yet exist — never overwrites an existing work plan. +# Feature branches: create AGENTS.md from AGENTS.develop.md plus the CLAUDE.md +# pointer when absent — never overwrites an existing work plan (plans are +# tracked on the branch and restored by git itself on switch). +# +# develop/master: these branches carry NO AGENTS.md/CLAUDE.md (dropped at +# feature finalization). Untracked local leftovers are deleted so a stale plan +# from a previous feature checkout cannot leak into integration-branch work. # # Installed via `git config core.hooksPath .githooks` — see .githooks/README.md. AGENTS_DEVELOP="AGENTS.develop.md" AGENTS_MD="AGENTS.md" +CLAUDE_MD="CLAUDE.md" # $3 is 1 for branch checkout, 0 for file checkout — only act on branch switches -if [ "$3" = "1" ] && [ -f "$AGENTS_DEVELOP" ] && [ ! -f "$AGENTS_MD" ]; then - cp "$AGENTS_DEVELOP" "$AGENTS_MD" - echo "[hook] Created AGENTS.md from AGENTS.develop.md" -fi +[ "$3" = "1" ] || exit 0 + +BRANCH=$(git branch --show-current 2>/dev/null) + +# Empty on detached HEAD (bisect/rebase steps) — no lifecycle there: files +# created on a detached HEAD would block later checkouts of any commit that +# tracks them ("untracked working tree files would be overwritten"). +# (On unborn HEAD git prints the branch name — harmless feature-branch +# treatment, self-heals on the next develop/master checkout.) +[ -n "$BRANCH" ] || exit 0 + +case "$BRANCH" in + develop|master) + for f in "$AGENTS_MD" "$CLAUDE_MD"; do + # only untracked leftovers — a tracked file is managed by git itself + if [ -f "$f" ] && ! git ls-files --error-unmatch "$f" >/dev/null 2>&1; then + rm -f "$f" + echo "[hook] Removed untracked $f (integration branches keep no agent files)" + fi + done + ;; + *) + if [ -f "$AGENTS_DEVELOP" ] && [ ! -f "$AGENTS_MD" ]; then + cp "$AGENTS_DEVELOP" "$AGENTS_MD" + echo "[hook] Created AGENTS.md from AGENTS.develop.md" + fi + if [ ! -f "$CLAUDE_MD" ]; then + printf 'Read AGENTS.md.\n' > "$CLAUDE_MD" + echo "[hook] Created CLAUDE.md pointer" + fi + ;; +esac diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 4add85b1..20698a74 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,7 +1,8 @@ #!/bin/bash -# pre-commit hook: format Rust code only. +# pre-commit hook: format Rust code + root-md allowlist guard. # -# Runs `cargo fmt` and stages any reformatting so CI's fmt-check can't fail. +# 1. Runs `cargo fmt` and stages any reformatting so CI's fmt-check can't fail. +# 2. Rejects commits that introduce a root-level *.md outside the allowlist. # # Installed via `git config core.hooksPath .githooks` — see .githooks/README.md. # @@ -27,4 +28,11 @@ if [ -n "$FMT_CHANGED" ]; then echo "pre-commit: staged rustfmt changes ($FMT_CHANGED)" fi +# --- root-md allowlist guard ------------------------------------------------- +# Branch-aware (AGENTS.md/CLAUDE.md are feature-branch-only, dropped at +# finalization). Shared with pre-merge-commit via lib/root-md-guard.sh — +# rationale and allowlist live there; see .githooks/README.md. +. "$(dirname "$0")/lib/root-md-guard.sh" +root_md_guard || exit 1 + exit 0 diff --git a/.githooks/pre-merge-commit b/.githooks/pre-merge-commit new file mode 100755 index 00000000..aa306513 --- /dev/null +++ b/.githooks/pre-merge-commit @@ -0,0 +1,11 @@ +#!/bin/bash +# pre-merge-commit hook — branch-aware root-md allowlist guard on the merge +# result. A CLEAN merge never invokes pre-commit, so without this gate +# AGENTS.md/CLAUDE.md could ride a clean feature→develop merge back onto the +# integration branch. +# +# Installed via `git config core.hooksPath .githooks` — see .githooks/README.md. + +. "$(dirname "$0")/lib/root-md-guard.sh" +root_md_guard || exit 1 +exit 0 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..d16566f4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +# Weekly dependency refresh — replaces reactive minimum-fix patching. +# Grouped so minor+patch ride one PR; majors stay individual for review. +# target-branch: dependabot always opens (and rebases) PRs against develop — +# without it the base resets to the repo default (master) on every rebase. +version: 2 +updates: + - package-ecosystem: cargo + directory: / + target-branch: develop + schedule: + interval: weekly + open-pull-requests-limit: 10 + groups: + cargo-minor-and-patch: + update-types: + - "minor" + - "patch" + + - package-ecosystem: github-actions + directory: / + target-branch: develop + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/workflows/agent-files-check.yml b/.github/workflows/agent-files-check.yml new file mode 100644 index 00000000..d029c7fc --- /dev/null +++ b/.github/workflows/agent-files-check.yml @@ -0,0 +1,55 @@ +name: Agent files check + +# AGENTS.md and CLAUDE.md are branch-local agent working files: tracked on +# feature branches while building, dropped at finalization (git rm before the +# PR — see AGENTS.develop.md, "For agents starting a new feature branch"). +# They must never reach develop or master. +# +# Locally the branch-aware pre-commit / pre-merge-commit hooks enforce this; +# this check is the server-side net for PRs merged via the GitHub UI or from +# clones without hooks installed (core.hooksPath unset = no hooks run at all). +# +# Visible PR check (red X), consistent with changelog-check: this repo's +# merges routinely go through `gh pr merge --admin`, which would bypass a +# required check just the same. No label bypass exists on purpose: the only +# correct fix is dropping the files from the PR head. + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [develop, master] + +permissions: + contents: read + +concurrency: + group: agent-files-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + no-agent-files: + runs-on: ubuntu-latest + steps: + # pin@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: PR head must not carry root AGENTS.md / CLAUDE.md + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + BAD="" + for f in AGENTS.md CLAUDE.md; do + if git cat-file -e "$HEAD_SHA:$f" 2>/dev/null; then + BAD="$BAD $f" + echo "::error::$f exists at the PR head — branch-local agent file, must not reach develop/master." + fi + done + if [ -n "$BAD" ]; then + echo "::error::Run the branch finalization step: git rm$BAD, commit, push." + exit 1 + fi + echo "No root AGENTS.md/CLAUDE.md at the PR head — good." diff --git a/.github/workflows/bump-develop.yml b/.github/workflows/bump-develop.yml index 5cde4d43..d195302c 100644 --- a/.github/workflows/bump-develop.yml +++ b/.github/workflows/bump-develop.yml @@ -26,17 +26,19 @@ name: Bump develop version # actions/create-github-app-token instead of the secret below. # Until CI_PAT exists, every run fails fast at checkout (empty token). +# `pull_request_target` (not `pull_request`): fork PRs run `pull_request` +# WITHOUT repo secrets, so CI_PAT is empty and checkout fails +# ("Input required and not supplied: token" — seen on fork PR #220). +# pull_request_target runs in base-repo context WITH secrets. Safe here: the +# workflow file comes from develop (never from the PR) and this job checks out +# `ref: develop` only — the PR's code is never executed or checked out. on: - pull_request: + pull_request_target: types: [closed] branches: [develop] -permissions: - contents: write - -# Serialize concurrent merges so each one gets its own bump. cancel-in-progress: -# false → a queued run waits, then re-reads a fresh develop (already carrying the -# previous bump) before bumping again. So N rapid merges → N patch bumps. +# Aikido 30640694: no workflow-level permissions; the single job below gets +# exactly what it needs (contents:write for the bump push). concurrency: group: bump-develop cancel-in-progress: false @@ -46,6 +48,8 @@ jobs: # Only when the PR was actually merged (closed-without-merge is a no-op). if: github.event.pull_request.merged == true runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout develop # pin@v4 @@ -56,6 +60,9 @@ jobs: # the "block develop" ruleset may push to develop. See SETUP note above. # Passing `token:` also persists those creds so `git push` below works. token: ${{ secrets.CI_PAT }} + # Aikido 35039595: explicit intent — this job DOES push the bump commit, + # so credentials must persist (see "How do I fix it?" guidance). + persist-credentials: true - name: Compute next patch version id: ver diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml new file mode 100644 index 00000000..43193d1a --- /dev/null +++ b/.github/workflows/changelog-check.yml @@ -0,0 +1,82 @@ +name: Changelog check + +# Enforces the CHANGELOG.md convention documented at the top of that file: +# every PR into `develop` is expected to add an entry under the current +# pending-version heading. This exists because it kept NOT happening — +# several PRs (git-hooks unification #193, the MSYS path-translation fix +# #196/#197) landed on develop with no changelog entry, and nobody noticed +# until a later /overview pass asked "wait, which bugs did we actually fix?" +# and the answer wasn't written down anywhere but commit messages. +# +# This is a visible PR check (red X + annotation), not a hard merge block: +# this repo's "block develop" ruleset requires 1 review, and PRs here are +# routinely merged with `gh pr merge --admin` (admin bypass), which would +# also bypass a "required status check" the same way. Making this check +# required in the ruleset would not add real enforcement on top of that +# pattern — it would just be a required check nobody's token is forced to +# wait for. What this DOES do: put a clear, hard-to-miss red mark on the PR +# and in `gh pr checks ` output, so the reason to skip a changelog entry +# has to be a deliberate `no-changelog` label, not silence. +# +# Bypass: add the `no-changelog` label to the PR for genuinely +# user-invisible changes (pure CI/tooling/docs-only churn, no behavior +# change). Created once via: +# gh label create no-changelog --color 999999 \ +# --description "Skip the CHANGELOG.md-touched CI check (docs/CI-only PR)" + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + branches: [develop] + +permissions: + contents: read + +concurrency: + group: changelog-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + require-changelog: + runs-on: ubuntu-latest + steps: + # pin@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Check CHANGELOG.md was touched (or PR is exempt) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: | + set -euo pipefail + + if printf '%s' "$LABELS" | grep -qiw "no-changelog"; then + echo "::notice::'no-changelog' label present — skipping CHANGELOG.md check." + exit 0 + fi + + # Diff from the MERGE BASE, not the raw base SHA: after a rebase or + # a develop-merge into the branch, the two-dot range includes + # develop's own commits — including the CHANGELOG.md touches every + # other PR was required to make — and the check would false-pass on + # a PR that adds no entry of its own. + MB=$(git merge-base "$BASE_SHA" "$HEAD_SHA") + CHANGED=$(git diff --name-only "$MB" "$HEAD_SHA") + if printf '%s\n' "$CHANGED" | grep -qx "CHANGELOG.md"; then + echo "CHANGELOG.md was updated in this PR — good." + exit 0 + fi + + { + echo "::error::This PR does not touch CHANGELOG.md." + echo "::error::Add an entry under the current pending-version heading" + echo "::error::(see the convention comment at the top of CHANGELOG.md)." + echo "::error::If this PR genuinely has nothing user-visible to record" + echo "::error::(pure CI/tooling/docs-only churn, no behavior change)," + echo "::error::add the 'no-changelog' label to this PR to bypass this check." + } >&2 + exit 1 diff --git a/.gitignore b/.gitignore index 08b24af6..dc6ced1a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,12 @@ criterion/ # Testing /test-repos/ +# Local-only markdown (diagnoses, plans, worklogs, test scenarios). +# The repo root keeps only its allowlisted .md files — see AGENTS.md +# "Root file hygiene (markdown)". Also covered by the `.*/` rule above; +# listed explicitly so the intent survives a future edit of that rule. +.docs/ + # codesearch database (local index, binary files) .codesearch.db/ test_tools.jsonl @@ -46,3 +52,6 @@ test_tools.jsonl # last on purpose — .gitignore is last-match-wins, and the `!.githooks/` negation # above would otherwise re-admit it. .githooks/customer-patterns.local + +# build.ps1 output logs +.tmp/ diff --git a/AGENTS.develop.md b/AGENTS.develop.md index 45ec1d5b..39c30d7b 100644 --- a/AGENTS.develop.md +++ b/AGENTS.develop.md @@ -3,16 +3,20 @@ This file is the **develop-branch reference** for all coding agents. It is committed on `develop` and merged from every feature branch. +`AGENTS.md` and `CLAUDE.md` are **branch-local working files**: committed on the +feature branch while building, dropped before the PR merges — they never appear +on `develop` or `master`. See *For agents starting a new feature branch* below. + ## Setup (once per machine) ```bash git config core.hooksPath .githooks ``` -This enables the `post-checkout` hook that automatically copies `AGENTS.develop.md` -to `AGENTS.md` whenever you switch to or create a branch where `AGENTS.md` does not -yet exist. After that, add your feature plan to `AGENTS.md` — it stays in the branch -and is never merged back to develop. +This enables the hooks: `post-checkout` auto-creates `AGENTS.md` (from this +file) and the `CLAUDE.md` pointer on feature branches, `pre-commit` runs +`cargo fmt` + the root-md allowlist guard (branch-aware, see below), `pre-push` +blocks direct pushes to `master`. See `.githooks/README.md`. ## For agents starting a new feature branch @@ -22,13 +26,26 @@ If `AGENTS.md` was not auto-created by the hook, create it manually: cp AGENTS.develop.md AGENTS.md ``` -Then replace the content under `## Plan` at the bottom with the work plan for this branch. +Then append your work plan as a `## Plan` section at the end of `AGENTS.md`. Leave the architecture sections intact — they provide context. -**At the end of the branch, before the PR:** -- Update the `## Active feature branches` section below (remove this branch) -- Add one line to `## Changelog highlights` -- Commit: `docs: update AGENTS.develop.md for features/xxx` +**Commit `AGENTS.md`/`CLAUDE.md` on the feature branch as you work** — the plan +travels through the PR and survives branch switches. + +**At the end of the branch, before the PR (the changelog moment):** +1. Update `## Active feature branches` below (remove this branch) +2. Add one line to `## Changelog highlights` +3. Add the CHANGELOG.md entry under the pending-version heading (functionality changes only — pure tooling/docs churn gets the `no-changelog` PR label instead; workflow changes like this one never go in the changelog) +4. Drop the branch-local files and commit: + `git rm AGENTS.md CLAUDE.md` → `chore: drop branch-local agent files` + +Enforced mechanically (forgetting step 4 cannot leak the files): +- `pre-commit` rejects introducing root `AGENTS.md`/`CLAUDE.md` on + `develop`/`master` (direct commits and conflict resolutions). +- `pre-merge-commit` runs the same guard on the merge result — a clean + merge never invokes pre-commit. +- The `agent-files-check` CI workflow flags any PR into `develop`/`master` + whose head still carries them. **No active work plan lives here.** Feature branches carry their own `AGENTS.md`. This file contains only architecture, conventions, and changelog. @@ -40,13 +57,13 @@ This file contains only architecture, conventions, and changelog. This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. - **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. -- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`, and to `/git pr create` / `/git merge`. NEVER target `master`. -- **`master`** only receives release merges from `develop` (cut at release time). -- **Merge style = merge commits** (`--merge`), not squash. Repo history is full of `Merge pull request #N`. -- **Review requirement** is enforced by a repo ruleset (not branch protection). As repo owner, override with `gh pr merge --merge --admin --delete-branch`. +- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`. NEVER target `master` (releases only, cut at release time). +- **Merge style:** feature/fix → `develop` = merge commits (`--merge`); `develop` → `master` release PR = **squash**, with `--body "$(scripts/release-coauthors.sh)"` so contributors stay credited on the default branch. +- **Review requirement** is enforced by a repo ruleset. As repo owner, override with `gh pr merge --merge --admin --delete-branch`. - Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. +- **Release squashes regress the merge-base** → a release PR may fail with "cannot be cleanly created" even when content is fine. Fix: cut `release/vX.Y.Z` off develop, run `git merge -s ours origin/master` in it, verify the diff vs master is the intended release delta, PR that into master. Never merge master into develop directly. -Common mistake: a subagent runs `/git pr create` with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. +Common mistake: a subagent creates a PR with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. --- @@ -60,8 +77,6 @@ Core stack: Tantivy (BM25 FTS) + arroy (HNSW vectors) + fastembed/ONNX (embeddin tree-sitter (AST chunking) + LMDB (persistent storage) + rmcp 1.5.0 (MCP protocol). Hybrid BM25 + vector search fused via RRF. ---- - ## MCP tools Five tools exposed to agents: @@ -77,8 +92,6 @@ Five tools exposed to agents: All tools return `scope_required` structured errors in multi-repo mode when no `project` or `group` is specified, with `available_projects`, `available_groups`, and `hint_for_agent`. ---- - ## Multi-repo serve mode `codesearch serve` starts an MCP HTTP server on `127.0.0.1:{port}` (default 39725). @@ -92,25 +105,18 @@ or per-group. - `Closed` — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity. **Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. -All opens update `last_access` so the reaper can track every open repo. +All opens update `last_access` so the reaper can track every open. **Fan-out rule:** `get_chunk` and group queries use `touch=false` → repos open as Warm only, no FSW spawned. Only explicit `project=` queries use `touch=true` → Warm→Write transition. ---- - ## TUI `codesearch serve` with a TTY starts an embedded ratatui TUI (repo table, status, CPU). Without TTY: headless, logs only. -`codesearch serve --no-tui` — suppress TUI even with TTY (e.g. to run serve in one terminal -and open the TUI separately in another). - -`codesearch serve tui [--url http://...]` — standalone TUI that connects to a running serve -instance via HTTP polling of the `GET /status` endpoint. Can be opened and closed independently. - ---- +- `codesearch serve --no-tui` — suppress TUI even with TTY +- `codesearch serve tui [--url http://...]` — standalone TUI via HTTP polling of `GET /status` ## HTTP endpoints (serve mode) @@ -123,62 +129,105 @@ instance via HTTP polling of the `GET /status` endpoint. Can be opened and close | `/repos/:alias/reindex` | POST | Incremental or force reindex (background) | | `/mcp` | GET/POST | MCP streamable HTTP endpoint | ---- +## Supported languages + +17 tree-sitter grammars (table in README.md). `find_impact` has SCIP symbol +precision for C# (bundled `scip-csharp`, resident helper via its `serve` +subcommand) and TypeScript (`npx scip-typescript`). Protobuf is text-aware +chunking only. -## Supported languages (tree-sitter AST chunking) +## Release artifacts -Rust, Python, JavaScript, TypeScript, C, C++, C#, Go, Java (9 languages). +GitHub Actions produces release binaries per tag (plain + `-with-csharp` +variants): `codesearch-{windows-x86_64.zip,linux-x86_64.tar.gz,macos-arm64.tar.gz}`. +Single-file native binaries, no runtime dependencies. macOS build is manual-trigger +only (expensive runners). --- -## Release artifacts +## Current state -GitHub Actions produces 3 release binaries per tag: +- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (`.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness: `build.rs` `+`. See `RELEASING.md`. +- **Validation:** `cargo check` for iteration → `cargo clippy --all-targets -- -D warnings` → `cargo test --lib --bins` before a branch is done. No `--release` builds during the fix loop. +- **Deploy:** `..\copy-to-common.ps1` builds + copies both binaries to `~/.local/bin/`. Stop serve first — a running `codesearch.exe` is file-locked on Windows. -``` -codesearch-windows-x86_64.zip -codesearch-linux-x86_64.tar.gz -codesearch-macos-arm64.tar.gz -``` +## Implemented features (orientation only — narratives live in CHANGELOG.md) -Single-file native binaries, no runtime dependencies. macOS build is manual-trigger only -(expensive runners). +- **Federation** — `codesearch remote add/rm/list/mounts`; `@peer` group fan-out; mounted projects addressable as `project=/`; TUI mount management with live status. Remote write verbs (`add`, `reindex --force`) need a read-write peer; `rm` is not durable across a cold start. +- **Cloud indexer-job split** — heavy build job uploads a snapshot, light serve restores it. DOCS repos are `repo_read_only` in `repos.json`; only `custom-kb` stays writable with a bounded incremental reindex per KB `git pull`. +- **`find_impact`** — SCIP-precise transitive call-sites (C#, TypeScript), ambiguity envelope, partial-results warnings, freshness fields. On `busy: true` sleep `retry_after_seconds` and retry the SAME call (busy is progress, never a reason to fall back to text search); `index_head_sha` vs `current_head_sha` drift is surfaced, never auto-reindexed. +- **REST mirrors** — read-only HTTP mirrors of the MCP tools incl. `/find-impact`. ---- +## ⚠️ Design constraint — never poll a federated peer -## Key conventions for agents +**A federated peer is NEVER contacted on a timer.** Each poll wakes the peer's +scale-to-zero replica, which then self-warms its full idle window (measured ~50% +duty cycle from zero searches). Shipped twice (PR #181/#184), reverted twice — +do not re-attempt. Local repos may be polled in the background; peers are +contacted only by a real federated tool call (activity poke) or the explicit +TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). -- **Branch from develop**, never from master. Feature branches: `features/`. -- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that - is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. -- **`cargo check` / `cargo clippy`** for iteration. Never `cargo build --release`. -- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. - OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to - `C:\WorkArea\AI\codesearch\instructions\`. -- **Path normalization**: all path comparisons must go through a single normalize utility. - Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree - `.git` file resolution have each caused subtle bugs in the past. -- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. - ~2GB memory during indexing is a known limitation. No local fix available until upstream - fastembed exposes `OrtArenaCfg`. -- **Git worktrees**: `find_git_root` returns the worktree directory itself (fixed). - Each worktree is a separate indexable repo. Groups can be used to search across worktrees - of the same base repo. +## Open TODOs + +- **Low severity:** literal-mode snippets for markdown chunks sometimes show the chunk's opening line instead of the matched line (the `match_info.unwrap_or_else` fallback), making true hits look like false negatives. --- +## Key conventions for agents + +- **Branch from develop**, never from master. Feature branches: `features/` (or `fix/`, `chore/`). +- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. +- **Always fix bugs you encounter — including pre-existing ones** the current branch did not introduce. Clean code and tests are the bar: a fix lands with a test that fails without it (reintroduce the defect, watch it fail — the proof rule under *Search errors*). +- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to `C:\WorkArea\AI\codesearch\instructions\`. +- **Root file hygiene (markdown)**: root markdown is allowlisted — `AGENTS.develop.md`, `README.md`, `README_CSharp.md`, `CHANGELOG.md`, `RELEASING.md` always; `AGENTS.md` and `CLAUDE.md` on **feature branches only** (branch-local, dropped at finalization — the pre-commit guard rejects them on `develop`/`master`). Anything else (diagnoses, plans, test scenarios, worklogs) goes into `.docs/` (gitignored, local-only). Enforced by the `pre-commit` root-md allowlist guard and the `agent-files-check` CI workflow. +- **Path normalization**: all path comparisons must go through a single normalize utility. Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree `.git` file resolution have each caused subtle bugs in the past. +- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. ~2GB memory during indexing is a known limitation. No local fix available until upstream fastembed exposes `OrtArenaCfg`. +- **Git worktrees**: `find_git_root` returns the worktree directory itself. Each worktree is a separate indexable repo. Groups can be used to search across worktrees of the same base repo. + ## Runtime locations - **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. -- **Build dir**: `target/release/` — this folder lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. +- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. +- **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. - **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. ## Deploying to runtime - `..\copy-to-common.ps1` — builds and copies **both** `codesearch.exe` and `scip-csharp.exe` to `~/.local/bin/` (the common execution dir). Use this to update the runtime binaries. **No `--release` builds — always dev/debug.** - The C# helper is built via: `dotnet publish helpers/csharp/scip-csharp.csproj -r win-x64 --self-contained -c Release` -- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true` which bundles everything into one exe. -- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location — only the single `.exe` is needed. +- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true`. +- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location. + +## Notes for agents + +- **Never use the bundled `codesearch` binary to investigate this repo** (it is the project under development). Use codesearch MCP tools first for discovery (this repo is indexed as `codesearch-git`); `grep`/`Read` for exact refs, other git refs, or when MCP returns nothing. +- **Tests live in sibling `_tests.rs` files**, table-driven preferred over near-duplicate per-case fns. +- **Tests that set env vars must be `#[serial]`** and set them via `crate::testing::EnvRestore` — cargo runs tests as parallel threads of one process, so an unserialised `set_var` races every reader. +- **Never call `.canonicalize()`** — use `safe_canonicalize()`. +- **Windows transient rename errors** (os error 5/32/33 from AV/Search-Indexer races): classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry. Never retry non-transient errors. +- **Counter-then-teardown races:** a background task tearing down state guarded by an in-flight counter must take the write lock BEFORE checking the counter and hold it across check + clear. Consumers increment the counter before acquiring the resource, so `counter == 0` under the write lock proves no consumer exists. + +### LMDB rules + +- **One `EnvOpenOptions::open()` per directory per process.** All access via `get_or_open_stores()` → `Arc`; SCIP opens share a per-directory env (`get_or_open_shared_env`). +- **Open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`) — heed refuses to reopen one path with different options. +- **Commit, never drop, a txn whose DB handle you keep** — an aborted txn's DBI is closed by LMDB; using it later yields a bare `EINVAL`. +- **A dropped `TrackedEnv` must close via `prepare_for_closing()`** — heed's `OPENED_ENV` cache keeps a clone, so a plain drop never runs `mdb_env_close` and Windows keeps the files locked for the process lifetime (the `index rm` os-error-32 bug). + +### Search errors must not become empty results + +Never `unwrap_or_default()` a store error on a search path — "no results" and "store down" must stay distinguishable. This defect was re-introduced across sibling handlers in seven review rounds; it is a class, not a site: + +- Render error chains with `{:#}`, never `{}`. Pass "not found" claims through `qualify_empty_result()`; never state a diagnosis you did not verify. +- The rule covers EVERY MCP handler: `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents`, and the single-store `project=` paths. +- **Warnings channels must terminate** on every path that writes to them, and the read must be reachable from the last write. +- **Every `for store in stores` fan-out opens its `*_warnings` channel before the loop.** `Err(_)` over a store result is banned: bind it, render `{e:#}`, carry it. `MultiReadOutcome` is `#[must_use]`. +- **Take the channel as a parameter** — use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. +- **New response shapes use the shared exits, not hand-rolled ones.** `serde_json::json!` renders `None` as an explicit `null`, so conditional keys must be *inserted*, not set. +- **Suppress `suggested_tool` when warnings are present.** +- **Verify a batch edit by re-running its detector over the whole file**, including the lines the edit added. +- **Before claiming a test pins a fix, reintroduce the defect and watch it fail.** +- **A caller-facing literal wrapped across lines needs a `\` continuation** — enforced by `tests/caller_facing_literals.rs`. --- @@ -186,13 +235,16 @@ Single-file native binaries, no runtime dependencies. macOS build is manual-trig | Branch | Description | |---|---| -| `features/symbol-references` | `find_impact` MCP tool, C# SCIP helper, blast-radius analysis | +| *(none)* | | --- ## Changelog highlights (recent) -- **v1.0.90** — `codesearch serve tui` standalone TUI, `--no-tui` flag, `GET /status` endpoint -- **v1.0.86** — Strict `get_chunk` scoping in multi-repo mode; zombie-proof idle reaper -- **v1.0.85** — `codesearch doctor` with `--all` and `--repo` flags -- **v1.0.84** — rmcp 0.9.1 → 1.5.0 (Claude Code 2.1.x protocol fix) +- **v1.3.37** — per-index embedding models end-to-end: serve queries, `POST /repos` and CLI index/stats/status honour the model each index records in its `metadata.json`; `serve --model` sets the default for newly created indexes; unrecorded indexes are queried with the built-in model plus a caller-facing warning; mid-rebuild indexes no longer report ready (PR #248) +- **v1.3.23–v1.3.36** — dependency + platform wave: rmcp 3.3, fastembed 6.1 + ort rc.13, tantivy 0.26, axum 0.8, ratatui 0.30 + crossterm 0.29, thiserror 2, notify 8, tree-sitter 0.27, dirs/sha2/scip/sysinfo refresh + dependabot (weekly); clears the open Aikido/RUSTSEC advisories +- **v1.3.19** — `find_impact` ambiguity envelope + `resolved_symbol`; partial-results `warnings`; C# symbol-key uniqueness (index v2.0) +- **v1.3.16** — REST `/find-impact` endpoint +- **v1.3.3** — federation hardening release + +Older entries: see `CHANGELOG.md`. diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 21b9cef9..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,100 +0,0 @@ -# AGENTS.md — codesearch - -_Last updated: 2026-08-05_ - -## Current state - -- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (CI via `.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness comes from `build.rs`'s `+` suffix. See `RELEASING.md`. -- **Validation:** `cargo check` for iteration, `cargo clippy -D warnings` for lint, `cargo test --lib --bins` before a branch is considered done. No `--release` builds during the fix loop — build only at the very end. -- **Deploy:** cloud peer runs the per-vendor federation split (one index per vendor sub-folder + custom-kb), image built locally via BuildKit `docker buildx --push`, all vendors reindexed and federation validated end-to-end (`project=cloud/`). - -## Implemented Features - -Release narratives live in `CHANGELOG.md`; this list keeps only the load-bearing facts. - -- **Federation peers** — `codesearch remote add/rm/list` (local `repos.json` peer config: `alias → url, api_key, group, into_group`) + `@peer` group references; `FederationClient` search/get_chunk fan-out with RRF. -- **Opt-in remote mount selection** — the `remote_mounts` allowlist in `repos.json` is the **single source of truth** for routing (`resolve_remote_project`), discoverability (`list_projects`/`scope_required`), TUI display, and `@peer` group fan-out (restricted to mounted projects, never the whole peer). Nothing a peer exposes is auto-mounted. CLI: `codesearch remote available|mount|unmount|mounts`. -- **Remote project mounting (1-to-1 passthrough)** — each mounted project is addressable locally as `project=/`; `FederationClient::search_project` forwards a single-project query straight to the peer. The TUI renders mounts in italic/cyan with a peer URL + live-status panel, and disables doctor/reindex/remove (those act on a local index a mount doesn't have). -- **Remote index management (`--remote`)** — `--remote ` on `index list/add/rm` plus an `index reindex` verb, driven through `FederationClient` (`ManagementOutcome`: `Ok` / `HttpError{status,reason}` / `Unreachable`). Endpoints: `GET /status`, `POST /repos {path}`, `DELETE /repos/:alias`, `POST /repos/:alias/reindex[?force=]`. `--json` on List/Reindex requires `--remote`. Without `--remote` every `index` verb is local and unchanged. -- **Cloud indexer-job split** — a heavy 4 vCPU/8 GiB build job uploads a snapshot; a light 1 vCPU/2 GiB serve restores it. The DOCS-read-only state is **enforced** by a per-repo `repo_read_only` flag in `repos.json` (set by the job's `mark_docs_readonly` step): serve's warmup opens those repos read-only and returns early, so no embedding happens on the replica. Only `custom-kb` stays writable and gets a memory-bounded incremental reindex (fire-and-forget `POST /repos/custom-kb/reindex`) after each KB `git pull` moves `HEAD`. The job also prunes ghost vendors before publishing. See `integrations/cloud/README.md`. -- **Language coverage** — 17 tree-sitter grammars (table in README). `find_impact` has SCIP symbol precision for **C#** (bundled `scip-csharp`) and **TypeScript** (`npx scip-typescript`, host-resolved). Protobuf is Niveau 1 (text-aware chunking on `message`/`enum`/`service`/`rpc`) only — no `scip-protobuf` emitter exists today. -- **Scale-to-zero-safe federation: a federated peer is NEVER polled on a timer** — ⚠️ **design constraint, do not "improve" this.** Background polling of *local* repos is fine; a *federated* peer must never be contacted on any cadence. The embedded TUI's discovery tick is **config-only** (`REMOTE_ROW_REFRESH_SECS` = 5s, zero HTTP): it rebuilds mounted-remote rows from the `remote_mounts` allowlist so mount/unmount edits and `l` reloads surface, and contacts nobody. A peer is contacted only by (a) an **activity poke** — a real federated tool call just hit it, detected via `remote_peer_activity` in `ServeState`, refreshing that one peer, never a fan-out — or (b) the explicit `i` info-overlay keypress. Idle mounts therefore render activity as `-`, which is the correct steady state, not a fault. **Rejected reasoning (was shipped twice, PR #181/#184, and reverted):** "polling no faster than the host's idle-suspend term is harmless." It is not — each poll *woke* the peer's scale-to-zero replica, which then self-warmed for its own full idle window (~1h), giving ~50% duty cycle on a peer nobody queried (measured: wakes 120/121/120 min apart, zero searches). Not keeping a peer awake past its suspend term is strictly weaker than not waking it, and the two windows are unrelated values anyway (local host vs. remote peer). -- **Standalone remote TUI auth** — `codesearch serve tui --url ...` resolves the API key from `repos.json` (`remotes.*.url` match) or a `--api-key` override and threads the authenticated client through every TUI action, with distinct errors for "no key configured" vs. "key rejected (401)". -- **Keep-warm ping observability + spurious-wake fix** *(branch `fix/federated-silent-poll-diagnosis`)* — the `keep_warm_url` self-ping loop logs every ping (`debug!` on success, `warn!` on failure) instead of discarding both outcomes, and warns at startup when the target host isn't this server's own bind host — **except on a wildcard bind** (`0.0.0.0` / `::`), where our externally-visible host is unknown so the comparison proves nothing; without that carve-out the warning fired on every cold start of the *only* deployment where keep-warm is correct (Azure binds `0.0.0.0`, target is the ingress FQDN), which just trains operators to ignore it. Rule lives in the testable `keep_warm_foreign_target` helper. Keep-warm also **requires a real recorded tool call**: the old `most_recent_tool_call().unwrap_or(start)` fallback meant any wake that wasn't a tool call (`/status` and `/healthz` don't call `record_tool_call`) made the replica self-warm for its whole idle window — reachable *only* when the wake wasn't real work, so its sole practical effect was rewarding spurious wakes (~11× amplification). Full diagnosis, with Azure Log Analytics ground truth: `DIAGNOSE_FEDERATED_KEEP_WARM.md`. -- **CLI aliases** — `ls` for `list` (`index`/`groups`/`remote`), `rm` for `remove`. `index rm ` resolves a registered alias before falling back to path interpretation. - -> ℹ️ **Remote write verbs** (`add`, `reindex --force`) require a read-write peer; the cloud peer rejects them (`--force` → HTTP 500 "could only be opened read-only; cannot force-reindex"). An **incremental** `reindex` (no `--force`) of an already-registered repo *does* succeed on the cloud peer — that is the custom-kb auto-refresh path. `list` is always safe. `rm` is not durable — the next cold start re-registers from the restored snapshot. - -## Open TODOs - -Single source of truth for outstanding codesearch work. - -### Cloud / infra — needs decision before pickup - -- [ ] **C1: Automate the manual `codesearch-indexer` trigger** — currently `triggerType: "Manual"`; every rebuild today is a human running `az containerapp job start` by hand. The 2026-07-04 batching fix (see "Historical context" below) means large batches can no longer crash anything, but staleness is still only resolved manually. Options, not yet decided (needs vendor content update-cadence info): - - **Schedule trigger** on the existing job (`az containerapp job update --trigger-type Schedule --cron-expression "..."`) — no new Azure resources, just a cron cadence. - - **Event-driven** (Event Grid on the blob source triggering job start) — more precise, needs a new Event Grid subscription + small trigger function/Logic App. - - The single-app redesign (C2) remains a separate, bigger follow-up. -- [ ] **C2: Single-app collapse redesign** (collapse indexer job + serve into one scalable app) — proposed design ready: - 1. `az containerapp update -n codesearch-serve --cpu 2.0 --memory 4Gi` — new revision, cold start (restore last snapshot, sync corpus, start incremental reindex in-process). - 2. Poll `GET /status` every ~10-15s (timeout e.g. 15 min) until **all repos report `"status": "warm"`** — replaces the fragile in-process `indexing`-flag/120s-timeout detection in `entrypoint.sh` that caused the 2026-07-04 crash. - 3. Once warm, trigger a snapshot upload (existing `upload_snapshot` logic). - 4. `az containerapp update -n codesearch-serve --cpu 1.0 --memory 2Gi` — new revision, cold start, restore-only. - - **Why the blob round-trip is unavoidable:** LMDB (mmap-based) is not safe on network-mounted volumes (Azure Files/NFS — mmap needs local POSIX byte-range locking a network share can't reliably provide). The index must live on local ephemeral disk, and ephemeral disk does not survive a Container Apps revision change (any `--cpu`/`--memory` update triggers one) — hence some durable handoff (blob snapshot) is structurally required. - - **Scoped first step shipped (2026-07-08):** incremental reindex in-process on serve is live — but *only* for the small **custom-kb** repo (`docker/entrypoint.sh`'s serve-mode KB pull loop fires `POST /repos/custom-kb/reindex` whenever `git pull` moves `HEAD`). Safe on the 1-2 GiB replica because incremental refresh is memory-bounded. The heavy DOCS corpus deliberately stays job-only. - - **Still open:** retire `codesearch-indexer` entirely or keep for DR; scheduled script vs Logic App vs wrapper CLI command (`codesearch cloud rebuild --remote `?). - -### Historical context (for C1/C2 above) - -**Fixed — incremental-refresh OOM crash-loop (2026-07-04):** `IndexManager::perform_incremental_refresh_with_stores` (`src/index/manager.rs`) used to chunk + embed the ENTIRE changed-file delta in one unbounded in-memory `Vec` before writing anything to the stores. A normal incremental delta was harmless; a vendor sync dropping thousands of files at once OOM'd the 1 vCPU/2 GiB `codesearch-serve` container, which then crash-looped. Fixed by batching: `changed_files.chunks(batch_size)` processed sequentially (chunk+embed+insert+commit per batch, single `build_index()` at the end), bounding peak memory to O(batch) regardless of delta size. Batch size defaults to `INCREMENTAL_REFRESH_BATCH_SIZE = 200` (`src/constants.rs`), override via `CODESEARCH_INCREMENTAL_BATCH_SIZE`. No test for the multi-batch path itself (existing `manager.rs` tests avoid real embedding, same reasoning as the gated `csharp_helper_integration` test) — verify end-to-end on a real large corpus if in doubt. - -This also explains an earlier cosmetic symptom: the `docs` repo's `/status` staying on `open`/`write` for 4+ minutes after a cold start (never blocked queries — search worked within ~10-25s of the replica becoming reachable). Root cause was the same unbounded-batch warmup path, not a separate status-tracking bug. - ---- - -## ⚠️ Branching & PR workflow (READ FIRST) - -This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. - -- **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. -- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`, and to `/git pr create` / `/git merge`. NEVER target `master`. -- **`master`** only receives release merges from `develop` (cut at release time). -- **Merge style = merge commits** (`--merge`), not squash. Repo history is full of `Merge pull request #N`. -- **Review requirement** is enforced by a repo ruleset (not branch protection). As repo owner, override with `gh pr merge --merge --admin --delete-branch`. -- Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. - -Common mistake: a subagent runs `/git pr create` with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. - -> **Note (2026-08-03):** the "merge commits, not squash" rule above is about feature/fix PRs into `develop`. Release PRs (`develop → master`) are, by contrast, squash-merged — which means master's release commits never become ancestors of develop. Over time this regresses `git merge-base(master, develop)` and can produce a false `CONFLICTING` mergeable state on a release PR even when the content is identical. If that happens, do not merge `master` into `develop` directly (history rewrite) — cut a throwaway `release/vX.Y.Z` branch off `develop`, run **`git merge -s ours origin/master`** in *that* branch (the merge **strategy** `-s ours`, *not* the option `-X ours`), verify the content diff is empty (`git diff origin/master`), and PR it into `master` instead. -> -> Why the strategy and not the option: against the regressed merge-base, `-X ours` still runs a real three-way merge that treats both sides' content as additions and drags master's stale lines in — a Frankenstein diff (`src/mcp/mod.rs` gained +333 stale lines this way on the v1.2.0 attempt). `-s ours` ignores master's tree entirely and keeps develop's content exactly, which is the desired result here (in this scenario develop's tree already equals master's content); the merge commit only exists to record master as a parent so the merge-base advances. Confirmed empirically on the v1.2.0 release: the `develop → master` PR #185 came back `CONFLICTING`; the throwaway `release/v1.2.0` branch built with `git merge -s ours origin/master` produced an empty content diff and merged clean (#186). - -## Notes for OpenCode / agents - -- **Validation:** `cargo check` and `cargo clippy` for iteration. No `--release` builds — always dev/debug until the very end. -- **Runtime:** `C:\Users\develterf\.local\bin\` — `codesearch.exe` + `helpers/csharp/scip-csharp.exe` -- **Build:** `target/release/` — outside repo (via `CARGO_TARGET_DIR`). `build.ps1` self-heals `core.bare=false` before invoking cargo — this checkout is a bare+working-tree hybrid whose `core.bare` intermittently resets to `true` (VS Code's git integration rewrites `.git/config` on ref changes), which makes cargo abort with `did not expect repo to be bare`. No need to flip it manually before building; `build.ps1` does it. -- **Deploy:** `..\copy-to-common.ps1` — builds + copies both binaries to `~/.local/bin/`. A running `codesearch.exe` is file-locked on Windows; stop serve before deploying. -- **Tests live in sibling `_tests.rs` files**, not in embedded `#[cfg(test)]` blocks inside `mod.rs` (mcp/serve/search/cache/db_discovery follow this). Prefer table-driven tests over near-duplicate per-case fns. -- **Canonical paths:** NEVER call `.canonicalize()` directly. Always use `safe_canonicalize()`. -- **Windows transient file errors:** `fs::rename` (and friends) can fail with raw OS errors 5/32/33 (ACCESS_DENIED/SHARING_VIOLATION/LOCK_VIOLATION) purely from an AV/Search-Indexer handle race — not a real conflict. Classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry (see `atomic_write_json`, the FTS commit retry in `fts/tantivy_store.rs`). Never retry non-transient errors. -- **LMDB rule:** No two `EnvOpenOptions::open()` on same dir in same process. All access via `get_or_open_stores()` → `Arc`. -- **LMDB rule — commit, never drop, a txn whose DB handle you keep:** any `open_database` / `create_database` whose handle outlives the opening transaction MUST end that transaction with `commit()`. `drop()` aborts, and LMDB closes handles opened in an aborted transaction. Storing a DBI from a dropped `RoTxn` yields a bare `EINVAL (os error 22)` on first use, with no other symptom. This shipped in `open_readonly` from the initial commit and only surfaced once read-only became a permanent mode, diagnosed and fixed in commit `8f62482`. -- **LMDB rule — open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`). heed refuses to reopen one path with different options, so a partial rollout turns a working reopen into an intermittent failure. -- **Search errors must not become empty results:** never `unwrap_or_default()` a store error on a search path. An empty result set and a failed store must stay distinguishable by the caller — "no results" is the most misleading signal this system can emit. Render error chains with `{:#}`, never `{}`; plain `{}` prints only the outermost `.context(...)` and hides the actual fault. - - **The rule covers EVERY MCP handler, not just `search`.** `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents` and the single-store `project=` paths all report store errors too. `Err` from `get_chunk` / `get_embedding` / `FtsStore::search` may never be matched as `Ok(None)`, `.ok()`, `unwrap_or_default()` or `if let Ok(..)` without an else. This defect was fixed in one sibling handler and left in the other three times across four review rounds; it is a *class*, not a site. - - **Never state a diagnosis you did not verify.** "not found" / "may not be indexed" is a claim about the corpus, and it is wrong when the store never answered. Pass such messages through `qualify_empty_result()`. - - **Carry failures in a type that cannot be dropped silently.** `MultiReadOutcome` is `#[must_use]` and yields results only via `into_results(&mut warnings, what)`. Reaching for `.results` and discarding `.failures` is `unwrap_or_default()` under a new name. - - **Do not suggest a retry against a store you know is down.** Suppress `suggested_tool` when warnings are present (`retry_hint()`). - - **A warnings channel must terminate, on every path that writes to it.** Every `*_warnings: Vec` in a handler has to end in either a `warnings` field on the response or a `qualify_empty_result()` call. A channel that is written but never read is invisible to clippy *and* to the tests — it looks fixed and behaves exactly as before. **Confirming a read site exists is not enough:** `similar_warnings` had one, inside an early-return arm, so every write after that arm was discarded. Check that the read is *reachable from the last write*, and say which response path carries it. - - **Verify a batch mechanical edit by re-running its detector, not by intent.** An edit heuristic that hit 5 of 9 sites is indistinguishable from one that hit 9 of 9 unless the post-condition grep comes back empty — and the detector must run over the *whole file including the lines the edit added*, or it will miss defects the fix itself introduced. - - **A new response shape needs a new shared exit, not a new hand-rolled one.** The channel has to be carried on the *success* path too, not only the empty one: a short-but-plausible list, or a confidently-returned single object, from a partially-dead group is the same false negative as an empty result and harder to notice. Item-list handlers do this via `respond_with_items()`; object-returning handlers add `Option> warnings` to their response struct (`GetChunkResponse`) or insert the key into their payload (`ambiguous_chunk_payload()`). Nine sites of this defect were found across seven rounds, six of them *after* the per-site fixes were reviewed and passed — per-handler discipline is what failed, so the fix has to be structural. Note the shape trap: `serde_json::json!` renders `None` as an explicit `null`, so a conditional key must be *inserted*, not set — otherwise the healthy path silently changes shape. - - **The rule starts at the fan-out, not at the channel.** Every `for store in stores { … }` whose body can produce an `Err` must open a `*_warnings` channel *before* the loop. `Err(_)` over a store result is banned outright: bind it, render it `{e:#}`, and carry it. A handler that discards the error with no channel is invisible to a grep for `*_warnings` **and** to clippy — which is exactly how `status(kind="index")` and `status(kind="projects")` survived eight rounds of hunting this class. - - **Take the channel as a parameter, not as a field the handler fills in.** A `warnings` field on a response struct is the obvious fix and the weaker one: the handler stays free to pass `None`, and a test that builds the struct itself cannot see it happen. Round 8 proved this — the round-7 defect was reintroduced at the `get_chunk` success path and all 630 tests still passed. Use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. - - **Before claiming a test pins a fix, reintroduce the defect and confirm it fails.** A green suite over a restored defect is the only proof that matters, and a test named after an acceptance criterion that constructs the response itself is testing serde, not the handler. Note `serde_json::Map` is a `BTreeMap` here (no `preserve_order`), so a `to_value` round-trip silently re-sorts keys — a healthy path must serialize the struct directly. - - **A caller-facing literal wrapped across lines needs a `\` continuation**, or the next line's indentation becomes part of the message. Enforced by `tests/caller_facing_literals.rs`, not by review: three commits shipped this defect through reviews that were explicitly hunting it, because the mangled text still satisfies every `contains(...)` assertion. A detector that only runs by hand gets skipped on exactly the commit that needs it. -- **Counter-then-teardown races.** A background task that tears down state guarded by an in-flight counter (idle-checker closing a connection, a reaper dropping a handle, a GC sweep clearing a slot) must take the state's write lock *before* checking the counter, and hold that lock across both the check and the clear. Checking the counter first and taking the write lock afterwards — even with no other statement between them — leaves a window in which a consumer can still acquire the resource and have it torn down mid-use once the write lock lands. The fix composes because of how consumers are structured: every consumer increments the counter (e.g. via an RAII guard created at function entry) *before* it takes the read lock to acquire the resource. That means a consumer already holding the resource has necessarily already incremented — so the checker seeing `counter == 0` under its own write lock proves no such consumer exists — and a consumer that has not yet read will simply block on the held write lock until the teardown (or the "already gone" check) has completed. Found in the MCP proxy's idle-disconnect feature (`src/mcp/mod.rs`, the `idle_ticker.tick()` arm in `run_mcp_client`): the original version read `in_flight`/`peer_state` before acquiring the write lock; fixed by moving the acquisition first, per the "counter-then-teardown races" review lesson. -- **Tooling:** never use the bundled `codesearch` binary to investigate this repo (it's the project under development). Use codesearch **MCP tools first** for discovery (server verified working; this repo indexed as `codesearch-git`). `grep`/`Glob`/`Read` stay correct for a specific git ref / fetched PR head (codesearch only indexes the on-disk working tree), exact literal matching, or when MCP returns nothing. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d243c3f..6b90526b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,150 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.4.0] - 2026-09-17 + +### Security + +- **Dependency refresh: h2, rustls, quinn-proto, zerovec-derive, moka.** A semver-safe `cargo update` lifts 126 packages within their existing requirements, clearing five open advisories without code changes: h2 0.4.15→0.4.19 (Aikido 41883526), rustls 0.23.42→0.23.45 (RUSTSEC-2026-0285), quinn-proto 0.11.16→0.11.18 (Aikido 41883527), zerovec-derive 0.11.3→0.11.6 (Aikido 41883531) and moka 0.12.15→0.12.16 (Aikido 41297220). fastembed/ort are deliberately kept at 5.17.3/rc.12 — 5.17.4 hard-requires the unstable ort rc.13 API break, which moves in its own PR. + +- **rmcp 1.8.0 → 3.3.0 — clears the five Aikido advisories on the MCP SDK (34247111).** Behavior-identical upgrade: the legacy `initialize` handshake and session semantics stay the default (`ProtocolVersion::LATEST` remains 2025-11-25); the 2026-07-28 stateless lifecycle is opt-in upstream and deliberately not enabled here. Code impact stayed small because the `#[tool_router]`/`#[tool_handler]` macros absorb the new MRTR response enums: `Content`/`RawContent` are now `ContentBlock`, the constructors-only impl takes `#[tool_router(allow_empty)]`, the stdio proxy's manual `call_tool` returns `CallToolResponse`, and content assertions drop the removed `.raw` projection. + +### Changed + +- **ort rc.13 + fastembed 5.17.4 (ONNX runtime refresh).** fastembed 5.17.4 hard-requires ort 2.0.0-rc.13, so both move together. rc.13 renamed the CPU execution provider (`CPUExecutionProvider` → `ort::ep::CPU`); the embedder's import and construction follow, builder chain unchanged (arena allocator still on). Scope note: ort-sys rc.13 still pins `lzma-rust2 ^0.15`, so the lzma-rust2 0.16.5 advisory (Aikido 37515813) remains open until upstream bumps its build dependency. + +- **Vector-index stack aligned on rand 0.10: arroy 0.5→0.8, heed 0.20→0.22.** The direct `rand` requirement moves to 0.10.2 — the major rmcp already uses — instead of pinning three rand majors side by side in the lock. arroy 0.8 and heed 0.22 follow because arroy 0.8 hard-requires both (`rand ^0.10.2`, `heed ^0.22.1`); no call-site changes were needed beyond silencing heed's `EnvFlags::NO_TLS` deprecation (same LMDB `MDB_NOTLS` flag, same behavior — the type-state `Env` migration is deferred). heed-types 0.21, roaring 0.11 and ordered-float 5 ride along transitively. Note: the rand 0.9.5 Aikido flagged rides the fastembed→hf-hub/tokenizers chain and clears with the fastembed 6 major, not with this change. + +- **Embedding stack majors: fastembed 6.1, hf-hub 1.0, ndarray 0.17.** Zero call-site changes — the codesearch embedder sits on fastembed's `ModelType`/`InitOptions` surface, which 6.1 kept stable. Scope note: fastembed 6.1 still resolves `image 0.25`, so the weezl 0.1.12/moxcms 0.8.1 advisories (Aikido 30640676/37515815) remain open until fastembed adopts image 0.26+; likewise rand 0.9.5 stays via hf-hub 0.5/tokenizers. + +- **tantivy 0.22 → 0.26 with graceful FTS reset.** The collector API changed so `TopDocs::with_limit` needs `.order_by_score()` at the three call sites. Because tantivy cannot open index files written by an older major, an unreadable FTS index is now wiped and recreated as a fresh empty index instead of failing the whole DB open — the FTS index is derived data, BM25 results rebuild on the next (re)index, and a warning is logged pointing at `codesearch index`. Vector search and all non-FTS paths are unaffected; pinned by a regression test that feeds the store a corrupt `meta.json`. + +- **TUI stack majors: ratatui 0.30, crossterm 0.29.** Zero call-site changes — the serve TUI sits on stable surface (`CrosstermBackend`, `Terminal`, `TableState`, `Paragraph`, `Layout`). This removes the last lru 0.12.5 path (ratatui's chain now carries lru 0.18.4; tantivy stays on the 0.16.4 that upstream 0.26 pins). + +- **axum 0.7 → 0.8.** The only breaking surface hit: path parameters changed syntax from `:param` to `{param}` — all route registrations (`/repos/:alias*`, `/chunk/:id`) and the shared `CHUNK_PATH` constant move to brace syntax, including the federation client's URL templating that derives from the same constant. Extractors, middleware and `axum::serve` compile unchanged; the serve + federation test suites exercise the rebuilt router end-to-end. + +- **thiserror 1.0 → 2.0 (direct).** Drop-in for all error enums (`#[error]`, `#[from]` unchanged); tantivy's chain still carries thiserror 1.x transitively until upstream moves. + +- **File-watch stack majors: notify 6.1 → 8.2, notify-debouncer-full 0.3 → 0.7.** The debouncer absorbed `Watcher` into `Debouncer` itself (`debouncer.watch()/unwatch()` replace `.watcher().watch()`), and root cache tracking is now automatic, so the explicit `cache().add_root()` call goes away. The watcher's cache type now follows upstream's per-platform recommendation (`RecommendedCache`: `FileIdMap` on Windows/macOS, `NoCache` on Linux — file-ID tracking is an internal rename-detection optimization; event mapping never reads file IDs directly). Also removes `mio 0.8.11` from the lock entirely (it rode the Linux-only inotify 0.9 path; notify 8 uses inotify 0.11). + +- **tree-sitter 0.26 → 0.27, tree-sitter-proto 0.4 → 0.6.** Zero call-site changes — the chunker sits on the stable `Parser`/`Language` surface and grammars load through the ABI-stable `LANGUAGE.into()` (`tree-sitter-language`) route, so all 17 grammar crates stay pinned while the core moves a major. + +- **Small majors batch: dirs 7, sha2 0.11, scip 0.10, sysinfo 0.39; dead `tower`/`tower-http` direct deps removed.** dirs/scip/sysinfo were drop-in. sha2 0.11's digest arrays no longer implement `LowerHex`, so the two hash-to-hex sites (`file_meta.rs`, `chunker/mod.rs`) hex-encode the digest bytes explicitly — output unchanged. `tower` and `tower-http` were declared as direct dependencies but never imported anywhere (CORS/trace middleware never wired in); removing them shrinks the direct dependency surface (both remain in the lock transitively via axum/reqwest/hf-hub, which is upstream's business). + +### Fixed + +- **Serve auto-recovers LMDB storage-format corruption with a sequential wipe + rebuild.** After the arroy 0.5→0.8 / heed 0.20→0.22 major upgrades, every repo whose on-disk database was written by the previous binary failed its symbol rebuild with `MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size` (observed on all C# repos after deploy). When a symbol rebuild now fails with that error class, serve wipes the repo's DB directory — closing the LMDB envs first via the same eviction sequence `remove_repo` uses, with the same bounded retry for transient Windows lock holders — and force-reindexes it through the existing force-reindex machinery, whose store-open path recreates everything on the new formats. Recoveries are queued and processed strictly **one repo at a time**: each rebuild runs a full CPU-bound embed pass, so parallel recoveries would thrash the machine. Read-only repos are skipped with a pointer to the owning writer. This closes the gap the tantivy FTS graceful reset (above) already covered on the FTS side — the vector/symbol stores now self-heal the same upgrade boundary instead of staying red until an operator force-reindexes by hand. + +### Fixed + +- **Federated chunk fetch works again — URL residue and project-scope routing (todo #153).** Two independent defects: (1) the peer URL for a `chunk_ref` fetch was built by replacing `{id` without the closing brace, so the constructed path carried a stray `}` (`/chunk/2058%7D`) that real peers answered with 400 Bad Request — axum's `{id}` parameter happily swallowed the stray brace into the captured value, which is exactly why the mock-based tests never caught it; the replacement now covers the full `{id}` placeholder, pinned by a test whose route echoes back the exact path it was hit on. (2) `get_chunk` with `project=/` plus a plain `chunk_id` died in local routing with "Unknown alias" — mounted remote projects are now routed through the same federated fetch search uses (local aliases still win a name clash), so both `chunk_ref` and `project=`+`chunk_id` forms work against remote peers. + +## [1.3.19] + +### Changed + +- **`find_impact` never silently picks a symbol or an adapter.** An ambiguous name (overloads, multiple definitions on one line) now returns a structured ambiguity envelope with sorted candidates instead of the shortest fuzzy match; an explicit `symbol_key` request field selects an exact candidate (mutually exclusive with `symbol_name` / `file`+`line`), and every resolved answer names its canonical key in the new additive `resolved_symbol` field. With more than one language index installed and no `language` given, the tool asks which one instead of silently using the first (todo #139). + +### Added + +- **`find_impact` surfaces partial results.** Index warnings — non-compiling C# projects, swallowed reference-resolution exceptions, `scip-typescript` non-zero exits with usable output — now persist (C#: alongside the cached refs in the same transaction; TS: in the index meta) and ride the answer as an additive `warnings` array instead of vanishing into the log, so consumers such as the `audit` binary's `impact`/`removals` never read incomplete evidence as zero (todo #139). + +- **Claude Code web-guard hook is topic/mount-scoped.** Only queries about the mounted products reach the web guard, and the retry cache is keyed per mount. + +- **`codesearch serve --model X` sets the default model for newly created indexes.** Previously `--model` was inert on `serve` (each repo's query model comes from its own `metadata.json`); now it is the model a repo is indexed with when it is added through serve without an explicit model — `POST /repos`, including the `codesearch index add` path delegated to a running serve — so an operator can make a non-default model the norm without passing `--model` on every add. An index that already records its own model is never overridden (an explicit `model` in the request still wins). Serve reports the default in `GET /status` as `default_model` and at startup. It is deliberately **not** the query fallback for a repo whose `metadata.json` records no model: a legacy index built before the model-recording contract is queried with the built-in 384-dim default, and the search response carries a warning naming the assumed model and the re-index command. Following the serve default there would break a legacy repo the instant an operator set `--model` to a model of another dimension (a 384-dim index queried with a 768-dim model fails), and degrade it silently for a same-dimension model. + +### Fixed + +- **`status` no longer reports `ready` for an index that cannot be searched.** Readiness keyed only off `total_chunks`, so a repo mid-rebuild — chunks inserted, `build_index()` not yet run — reported `status: "ready"` / "Index is ready for searching." while every search failed with `Index not built. Call build_index() after inserting chunks.` Both the single-repo and group status paths now require the vector index (`stats.indexed`) to be built before reporting `ready`, otherwise they report `building` with a message that says the vector index is not built. A store that failed to report stats is still surfaced separately (degraded-ready with `warnings`), never misread as "not built". + +- **Adding a repo with `--model` now creates the index at that model's dimension.** `POST /repos` — the path `codesearch index add --model …` delegates to when serve is running — opened the store with the default 384-dim dimension and applied the model override only to `metadata.json` afterwards. The background reindex then embedded 768-dim EmbeddingGemma vectors into a 384-dim store and indexed nothing: the `.codesearch.db` directory existed, `stats` showed the new dimension, and no files were indexed. The store is now opened at the override's dimension. + +- **The CLI no longer downgrades a non-default index to the 384-dim default.** `codesearch index` resolved its embedding model as `--model`-or-`ModelType::default()` and never consulted the model recorded in the index's `metadata.json`, so re-indexing a repo built with EmbeddingGemma embedded 384-dim MiniLM vectors against a 768-dim store (and `FileMetaStore` logged "Model changed, full re-index required", wiping the file metadata). The CLI now resolves the recorded model through the same helper the serve/watcher paths use; an explicit `--model` that disagrees is rejected with a pointer to `--force`. Relatedly, `codesearch stats`, `get_db_stats` and the repo listing opened the vector store with a hardcoded 384, so `Dimensions:` always read 384 for every index — they now read the recorded dimensions. + +- **`status(kind="index")` reports the routed repo's model, not the service default.** The `model` field was the service's own model (the hardcoded default in serve mode) while `dimensions` came from live store stats, so every repo read `minilm-l6-q` regardless of what it was indexed with. It now resolves per repo for `project=`, and reports the common model — or `mixed` — for a group. + +- **Serve mode embeds each query with its target repo's indexed model.** The multi-repo MCP service built its shared embedder from `ModelType::default()` (384-dim MiniLM) and ignored both `--model` and the `model_short_name` each index records, so every semantic query against an index rebuilt with a 768-dim model (EmbeddingGemma) failed with `Query embedding dimension mismatch: expected 768, got 384` — and a same-dimension mismatch would have silently compared incomparable vector spaces. The service now resolves the model per routed repo (the same `metadata.json` contract the indexing path already followed) through a per-model `EmbeddingServicePool`; group fan-out embeds the query once per distinct model and searches each store with its own. (See the `serve --model` default for newly created indexes under **Added**.) + +- **Persisted helper-exit warnings are platform-stable — Linux CI green again.** `ExitStatus`'s `Display` renders `exit code: N` on Windows but `exit status: N` on Unix, so the non-zero-exit warning the TypeScript symbol indexer persists into its meta table disagreed with its own regression test on Linux, failing `test-linux`/`csharp-integration-tests` deterministically since #238. A shared `exit_status_text()` renders `exit code: N` on every platform (signal-terminated processes fall back to the platform string), applied to the persisted TS warning and the C#/TS helper log lines, pinned by a cross-platform unit test. + +- **C# canonical symbol keys no longer collapse distinct declarations.** Generic arity (``M`1``), the full containing-type chain and fully qualified parameter types are part of the key again, so overloads that previously shared one identity keep their own. The index version is bumped to 2.0 with a key-format stamp in the index meta: a stale-format index reports as absent and rebuilds once on upgrade (todo #139). + +## [1.3.16] + +### Added + +- **REST `/find-impact` endpoint (HTTP mirror of the `find_impact` MCP tool).** The read-only REST surface (`/search`, `/find`, `/explore`, `/chunk/:id`) now also mirrors `find_impact`: POST a `FindImpactRequest` body (`symbol_name`, or `file`+`line`; optional `language`, `project`, `group`) and receive the tool's JSON payload — busy envelope and `index_head_sha`/`current_head_sha` freshness fields included. Same auth class as the other REST mirrors: open on localhost binds, bearer key on network binds. Lets non-MCP clients — notably the `audit` binary — consume SCIP reference evidence without an MCP session. + +## [1.3.15] + +### Added + +- **`edit-guard` — a fourth Claude Code guard hook: edits now require a codesearch consultation first.** On `Edit`/`Write`/`MultiEdit` against a file in a codesearch-registered repo, the hook denies the edit unless codesearch was consulted for that exact path within the last 5 minutes: `find_impact` for SCIP-backed languages (`.cs .ts .tsx .mts .cts`), `find(kind="usages")` for everything else. Markers are recorded by `edit-guard-post`, the first Claude Code `PostToolUse` hook in this repo: it fires on every `find_impact` call and every `find(kind="usages")` (kind check done script-side — matchers only see tool names), counts any outcome ("no results" and "no SCIP backend" included, so the guard can never wedge permanently), and prunes expired entries on write. The guard accepts ANY marker for the path within the window and fails open on unregistered repos, non-git paths and a crashed hook; missing/corrupt state counts as not consulted (deny on covered repos, allow everywhere else). Shared target-resolution/coverage helpers moved into `codesearch-common.sh/.ps1` (grep-guard sources them too; its PowerShell twin is thereby ported off the last `.codesearch.db`/Windows-only-path coverage signals, closing the #199 gap). The native installer writes the new scripts plus a `PostToolUse` registration, idempotent by exact command as before; the subagent preamble gained an EDIT RULE line. Hook self-tests: `bash integrations/claude-code/hooks/run-tests.sh` (todo #134). + +## [1.3.14] + +### Fixed + +- **Concurrent cold opens no longer wedge a repo behind the LMDB double-open guard.** Two overlapping first opens of the same repo (e.g. a `find_impact` racing its own retry) could both reach `try_open_stores`; the loser tripped the double-open guard and cached `Conflicted`, which the self-heal could never cure while the winner held its env — the repo stayed broken until a serve restart (2026-09-08 incident, todo #131). Cold opens are now single-flight per alias in `ServeState`: the loser waits on a per-repo lock and then hits the winner's cache entry. Covers both cold-open entry points (`get_or_open_stores`, `warmup_repo`); the fast path stays lock-free. + +## [1.3.13] + +### Fixed + +- **scip-csharp stderr no longer scrambles the serve TUI.** The resident `scip-csharp serve` helper inherited the host's stderr, so MSBuild workspace diagnostics (e.g. project-load failures from stale NuGet restore state) were sprayed raw over the TUI, bypassing the file-only serve logger. Helper stderr is now piped and drained through tracing, with helper `[WARN]`/`[Failure]` lines classified as warn level across all invokers (index, find-refs, batch-find-refs, serve) — workspace-load errors land in the codesearch log and surface as warnings instead of corrupting the terminal. + +## [1.3.10] - 2026-09-03 + +### Added + +- **Resident SCIP helper for `find_impact` (C#).** `scip-csharp serve` loads the Roslyn workspace once and answers find-refs requests over a JSON-lines protocol; a `WorkspacePool` caps resident workspaces (default 2, per-workspace heap cap, idle teardown). `find_impact` consults the pool first and keeps the one-shot spawn as fallback. +- **Local MCP ownership and readiness controls.** `codesearch mcp --mode local --readonly` serves an index without taking the writer lock or starting refresh/file-watcher tasks; `--require-ready` refuses to start on a missing, empty, partial or otherwise unusable index. Both fail explicitly outside local mode, where they cannot govern a remote serve process. + +### Changed + +- **`find_impact` stays responsive and truthful on cold caches.** Lookups run under a configurable budget; on overrun a structured busy answer (`retry_after_seconds`) is returned while the lookup continues in the background, so a retry gets progress or the warm result. Failures are typed (`failed`/`stale`) with actionable hints, and results carry `index_head_sha` vs `current_head_sha` so index drift is surfaced instead of guessed at. +- **`find` kind=`usages` discloses its lexical nature.** Results are re-ordered code-first before the limit is applied, C#/TS hits carry a `note` naming the `find_impact` upgrade path, and the tool description states the caveat outright. +- **build.ps1 writes cargo output to a repo-local `.tmp/build-.log`** (printed afterwards) and warns — never kills — when cargo/rustc processes are already running. + +### Fixed + +- **Local stdio MCP no longer masks search errors with `LMDB double-open prevented`.** A failed read on the live shared store preserves the original error instead of attempting a second database open. +- **SCIP symbol index: concurrent queries and rebuilds no longer fail each other.** All SCIP opens share one LMDB environment per index directory, and error states are cleared on eviction/force-reindex so the TUI no longer shows a permanently frozen failure. +- **`index rm` on Windows (os error 32).** Dropping the tracked LMDB environment now actually closes the heed env, so `data.mdb`/`lock.mdb` are released immediately instead of staying locked for the process lifetime. + +## [1.3.3] - 2026-08-18 + +### Added + +- **Pre-commit hook enforces the root-md allowlist**: a commit adding a root-level `*.md` outside the allowlist is rejected with a pointer to `.docs/`. Stray root docs and the tracked `docs/` folder were dissolved into gitignored `.docs/`. + +### Fixed + +- **grep-guard resolves Grep coverage from the search target and serve-hub registration** instead of the hook's cwd and `.codesearch.db` presence: absolute POSIX paths are detected correctly, coverage follows `repos.json` registration (a nested unregistered clone counts as uncovered), and the resolver fails open. +- **`index rm` against a running serve completes the file delete without stopping serve.** The DELETE client gets its own derived 80s timeout, and the retry loop awaits in-process LMDB holder drain instead of backing off blindly. +- **`index rm` no longer claims "DB deleted" when serve could not delete the files** — a `db_deleted: false` response surfaces as a warning naming the leftover directory and the recovery path. + +## [1.3.0] - 2026-08-15 + +### Added + +- **`GET /indexing?path=...`** — per-repo freshness probe (`covered`/`indexing`), and the grep-guard hook now waits-and-retries while a branch-switch reindex is in flight instead of forcing a grep fallback. Repo resolution follows the grep target rather than the hook's cwd. +- **CI checks that every PR into `develop` touches `CHANGELOG.md`** (visible-not-blocking; deliberate skips require the `no-changelog` label). Env-mutating tests are now `#[serial]` with panic-safe restore. + +### Fixed + +- **Federated peers retry transient scale-to-zero responses (502/503/504, bounded)** inside the active tool call — a cold-starting peer surfaces as a short delay with a clear "retry in ~30s" hint instead of a raw non-JSON error. Non-transient statuses and transport errors are not retried. +- **`index rm` end-to-end acceptance path pinned by a real integration test**: CLI → health probe → DELETE → serve deletes the DB dir without being stopped → clean "Unknown alias" afterwards. + ## [1.2.10] - 2026-08-12 ### Fixed -- **Caller-supplied MSYS POSIX paths (`/c/Users/...`) no longer silently create junk `:\c\Users\...` directories on Windows (#196, #197).** When an agent (or any non-MSYS caller — CLI, MCP client, `codesearch serve --register`) passed a POSIX-style drive path like `/c/Users/foo`, Rust on Windows resolved the leading `/` as "rooted on the *current drive*", i.e. `:\c\Users\foo`, creating orphan directories like `C:\c\Users\...` and silently indexing the wrong project. This is the path-pollution defect behind the orphan `-propagate-tmp` indexes (diagnosed end-to-end: an agent-created staging folder was indexed under both a real path and a polluted `C:\c\...` mirror that nothing ever cleaned up). Two new helpers in `src/cache/file_meta.rs`: `translate_msys_path` (Windows-only rewrite of `/c/...` → `C:/...` for a single ASCII letter after a leading `/` followed by `/` or EOL; idempotent on every other input; no-op on non-Windows where `/c/...` is a legitimate absolute path) and `normalize_user_path` (composes `translate_msys_path` + `strip_unc_prefix` — the single helper for every `safe_canonicalize(...).unwrap_or_else(_)` fallback site, per the repo's "structural fix" rule for the warnings-channel defect class). `safe_canonicalize` itself now calls `translate_msys_path` *before* canonicalising, so the success path is also covered structurally — not just the fallback. Applied at every user-supplied path boundary (11 production sites): `db_discovery/repos.rs` (`register`, `register_with_alias`, `unregister_path`, `alias_for_path`, `scan_for_remote`), `db_discovery/mod.rs` (`resolve_database_with_message`), `index/mod.rs` (the three `try_delegate_*_to_serve` functions + both `normalize_for_cmp` closures), and `serve/mod.rs` (`run_serve`'s `--register` loop). Non-repo indexing stays supported — intentionally NO git-repo check was added; the defect was purely about path resolution. Comprehensive regression tests pin both branches (existing-path success path + non-existing-path fallback, the actual defect site), plus register/unregister symmetry, plus Unix no-op guard. -- **A repo that failed to open once stayed broken until `serve` was restarted — a cached conflict is no longer replayed forever.** When opening a repo's database failed — typically a transient write lock, e.g. an indexing run holding the DB at the moment a query arrived — `ServeState` cached `RepoState::Conflicted`, and the fast path in `get_or_open_stores` replayed that error on every later call without ever retrying the open. The state's only documented exit was idle eviction, and that exit was unreachable: `evict_idle_repos` iterates `last_access`, but both paths that mark a repo Conflicted (`warmup_repo` and the `get_or_open_stores` slow path) propagate the failure with `?` *before* reaching their `touch_access` call, so a conflicted repo never gets a `last_access` entry and is never considered for eviction — however long it sits idle. Querying it did not help either: the fast path replayed the cached error while calling `touch_access` on the way, so the only queries that would have registered the repo for eviction were also the ones resetting its idle timer. Net effect: a momentary lock became indistinguishable from permanent corruption, curable only by restarting serve, while the error text promised the opposite ("the next query will retry automatically"). A cached conflict is now dropped on the next access and the open genuinely retried — cheap when it still fails, since that is just a refused file lock. This mirrors the missing-DB path, which already refused to cache `Conflicted` for the same reason. Regression test asserts recovery *without* a restart or an idle wait, and was confirmed to fail before the fix. -- **A federated peer is now never polled on a timer — the two 1.2.0 "scale-to-zero" fixes did not actually stop the cloud peer being woken.** 1.2.0 replaced the TUI's hardcoded 30s peer poll with the local serve's own `idle_suspend_secs` cadence and suppressed the startup poke, on the theory that polling no faster than the host's suspend term is harmless. It is not, and the release notes above overstated the fix. Measured on the deployed Azure Container Apps peer over a period with **zero** federated searches: wakes exactly **120/121/120 minutes** apart, each warm period **~67 min** — roughly a 50% duty cycle on an index nobody queried, each wake additionally paying an `azcopy sync` of the docs blob and a KB `git pull`. Two independent defects combined. **(1) The trigger:** the poll *itself* was the ingress traffic that woke the replica. Not keeping a peer awake *past* its suspend term is strictly weaker than not *waking* it, and the two windows were unrelated values anyway — the cadence read the **local** host's 2h default, not the peer's ~1h (which is why the 120-minute spacing, not 60, is the tell). **(2) The amplifier:** the cloud keep-warm loop fell back to the process start time when no tool call was recorded (`most_recent_tool_call().unwrap_or(start)`), and since `/status` and `/healthz` never call `record_tool_call`, any non-tool-call wake made the replica self-ping every 120s for its whole idle window — ~11× amplification. That fallback was unreachable in the case it was written for: a real tool call always records itself, so it could only ever fire when the wake was *not* real work. Now: the TUI's discovery tick is **config-only** (5s, zero HTTP) and merely rebuilds mounted-remote rows from the `remote_mounts` allowlist so mount/unmount edits and `l` reloads still surface; a peer is contacted only by an **activity poke** (a real federated search/get_chunk just hit that peer, so it is demonstrably already awake — single-peer, never a fan-out) or the explicit `i` info-overlay keypress. Keep-warm requires a real recorded tool call and otherwise lets the host suspend the replica. A peer staying warm for an hour *after real use* is correct and unchanged. Idle mounts render activity as `-`, now the normal steady state rather than a fault. Also fixed: removing the *last* peer from `repos.json` left its rows on screen forever (the snapshot was gated on a non-empty peer list), and the 1.2.0 "keep-warm target isn't self" warning false-positived on the only deployment where keep-warm is correct (the process binds `0.0.0.0` while the target is the ingress FQDN — a wildcard bind means the external host is unknown, so the check now stays silent). Background polling of **local** repos is unchanged and unaffected: the local/federated split is a deliberate design constraint. -- **`MDB_MAP_FULL` fatal crash on large corpora — LMDB mapsize cap raised + persistent embedding cache now auto-resizes too (#189).** Indexing a large corpus (e.g. a 1GB / 53k-file cargo-registry source producing >1.2M chunks) could crash with `MDB_MAP_FULL: Environment mapsize limit reached` once the vector store's auto-resize (already in place since an earlier fix) hit its old 8GB hard cap. Two changes: (1) the cap is raised to 16GB by default, and made runtime-overridable via `CODESEARCH_MAX_LMDB_MAP_SIZE_MB` (clamped to at least 1GB) for corpora that legitimately need more; (2) the **persistent embedding cache** (`~/.codesearch/embedding_cache//`) previously had no resize logic at all — it hit the same `MDB_MAP_FULL` on a hardcoded 512MB cap and silently degraded to a WARN-and-continue path, turning every subsequent embedding into a full ONNX-inference cache miss. It now retries with the same doubling-resize pattern as the vector store (up to 3 attempts, capped at the same runtime limit), persisting the grown size to `metadata.json` so a restart reopens at the correct size. When either store's cap is genuinely exhausted, the error/warning message now names the env var that raises it, instead of just reporting the size. -- **`build.ps1` now self-heals `core.bare=false` before invoking cargo.** This repo lives at `codesearch.git` as a bare+working-tree hybrid — a full checked-out source tree + `.git/index`, but `core.bare=true` in `.git/config`. `core.bare` intermittently resets to `true` (VS Code's git integration rewrites `.git/config` on ref changes; smoking gun: `github-pr-owner-number` duplicated 7× for `develop`), and when it does, cargo's source fingerprinting aborts every build with `did not expect repo ...\.git to be bare`, breaking `copy-to-common.ps1` → `build.ps1` → `cargo build`. `build.ps1` now forces `core.bare=false` right after `Set-Location`, before any cargo invocation. Idempotent and harmless for a normal (truly non-bare) checkout; non-fatal if git is unreachable. +- **MSYS POSIX paths (`/c/Users/...`) no longer create junk `:\c\Users\...` directories on Windows.** `translate_msys_path` + `normalize_user_path` are applied at every user-supplied path boundary; fixes the orphan `C:\c\...` index pollution. +- **A repo that failed to open once no longer stays broken until restart** — a cached `Conflicted` state is dropped on the next access and the open is genuinely retried. +- **Federated peers are truly never polled on a timer.** The 1.2.0 cadence fix still woke the cloud peer (the poll itself was ingress traffic, plus a keep-warm fallback amplified non-tool-call wakes). The TUI discovery tick is now config-only (zero HTTP); peers are contacted only by a real tool call or the explicit info overlay. +- **`MDB_MAP_FULL` on large corpora** — LMDB mapsize cap raised to 16GB (runtime-overridable), and the persistent embedding cache now auto-resizes instead of silently degrading to cache misses. +- **build.ps1 self-heals `core.bare=false`** before invoking cargo (VS Code's git integration intermittently flips the hybrid checkout bare, aborting every build). ## [1.2.0] - 2026-08-03 diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 5d302681..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -Read AGENTS.md. diff --git a/Cargo.lock b/Cargo.lock index 0b4d92ce..0dd98012 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -49,6 +49,15 @@ dependencies = [ "equator", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -57,9 +66,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -126,6 +135,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -160,22 +178,24 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arroy" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc5f272f38fa063bbff0a7ab5219404e221493de005e2b4078c62d626ef567e" +checksum = "2f88e18ede2cb2f69e1a68ea804c8fc2f19f748da5f28c795271f279f4068afb" dependencies = [ "bytemuck", "byteorder", + "enum-iterator", "heed", - "log", "memmap2", "nohash", - "ordered-float", - "rand 0.8.7", + "ordered-float 5.5.0", + "page_size", + "rand 0.10.2", "rayon", "roaring", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.20", + "tracing", ] [[package]] @@ -189,13 +209,22 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", +] + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", ] [[package]] @@ -225,7 +254,7 @@ dependencies = [ "num-traits", "pastey 0.1.1", "rayon", - "thiserror 2.0.19", + "thiserror 2.0.20", "v_frame", "y4m", ] @@ -255,9 +284,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -265,9 +294,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -278,13 +307,13 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.9" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "async-trait", "axum-core", "bytes", + "form_urlencoded", "futures-util", "http", "http-body", @@ -297,8 +326,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustversion", - "serde", + "serde_core", "serde_json", "serde_path_to_error", "serde_urlencoded", @@ -312,19 +340,17 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ - "async-trait", "bytes", - "futures-util", + "futures-core", "http", "http-body", "http-body-util", "mime", "pin-project-lite", - "rustversion", "sync_wrapper", "tower-layer", "tower-service", @@ -343,6 +369,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -358,6 +390,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit_field" version = "0.10.3" @@ -372,9 +419,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -397,6 +444,19 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -406,6 +466,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -415,13 +484,37 @@ dependencies = [ "objc2", ] +[[package]] +name = "bon" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "bstr" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", + "regex-automata", "serde_core", ] @@ -437,6 +530,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "by_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" + [[package]] name = "bytemuck" version = "1.25.2" @@ -448,13 +547,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -475,12 +574,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "cast" version = "0.3.0" @@ -498,9 +591,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.3.0" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", "jobserver", @@ -528,12 +621,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -580,9 +673,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -590,9 +683,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -602,21 +695,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmake" @@ -629,7 +722,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.2.10" +version = "1.4.2" dependencies = [ "anyhow", "arroy", @@ -638,46 +731,45 @@ dependencies = [ "bincode", "chrono", "clap", - "colored", + "colored 2.2.0", "criterion", "crossterm", "ctrlc", "dashmap", - "dirs 5.0.1", + "dirs 7.0.0", "fastembed", "fs2", "heed", - "hf-hub 0.3.2", + "hf-hub 1.0.0", "ignore", "indicatif 0.17.11", "libc", "moka", - "ndarray 0.16.1", + "ndarray", "notify", "notify-debouncer-full", "num_cpus", "ort", "pretty_assertions", "protobuf", - "rand 0.8.7", + "rand 0.10.2", "ratatui", "rayon", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "schemars", "scip", "serde", "serde_json", - "sha2", - "sysinfo", + "serial_test", + "sha2 0.11.0", + "sysinfo 0.39.6", "tantivy", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.20", "tokio", "tokio-util", - "tower", - "tower-http", "tracing", "tracing-appender", "tracing-subscriber", @@ -726,27 +818,22 @@ dependencies = [ ] [[package]] -name = "combine" -version = "4.6.7" +name = "colored" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "bytes", - "memchr", + "windows-sys 0.61.2", ] [[package]] -name = "compact_str" -version = "0.8.2" +name = "combine" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "static_assertions", + "bytes", + "memchr", ] [[package]] @@ -773,27 +860,63 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.0", + "unicode-width", "windows-sys 0.59.0", ] [[package]] name = "console" -version = "0.16.4" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ "encode_unicode", "libc", - "unicode-width 0.2.0", + "unicode-width", "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9603f79528ece8163c496f8932121cb36cfe46259e9c907bb3d8205139d7caa3" +dependencies = [ + "typewit", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" dependencies = [ "percent-encoding", "time", @@ -844,6 +967,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -855,43 +993,42 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -899,28 +1036,34 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -928,39 +1071,41 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "crossterm_winapi", - "mio 1.2.2", + "derive_more", + "document-features", + "mio", "parking_lot", - "rustix 0.38.44", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -991,6 +1136,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csscolorparser" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" +dependencies = [ + "lab", + "phf", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -998,10 +1172,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", - "nix", + "nix 0.31.3", "windows-sys 0.61.2", ] +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + [[package]] name = "darling" version = "0.20.11" @@ -1014,12 +1194,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -1038,15 +1218,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1062,13 +1242,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.23.0", + "darling_core 0.24.1", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1094,11 +1274,23 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + +[[package]] +name = "deltae" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" + [[package]] name = "der" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ "pem-rfc7468", "zeroize", @@ -1144,6 +1336,28 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "diff" version = "0.1.13" @@ -1156,17 +1370,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", ] [[package]] -name = "dirs" -version = "5.0.1" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "dirs-sys 0.4.1", + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", ] [[package]] @@ -1175,19 +1391,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys 0.5.0", + "dirs-sys", ] [[package]] -name = "dirs-sys" -version = "0.4.1" +name = "dirs" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.6", - "windows-sys 0.48.0", + "dirs-sys", ] [[package]] @@ -1198,7 +1411,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.2", + "redox_users", "windows-sys 0.61.2", ] @@ -1208,7 +1421,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "block2", "libc", "objc2", @@ -1216,13 +1429,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1236,9 +1449,9 @@ dependencies = [ [[package]] name = "downcast-rs" -version = "1.2.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "doxygen-rs" @@ -1263,9 +1476,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "encode_unicode" @@ -1275,25 +1488,51 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" dependencies = [ "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] -name = "equator" -version = "0.4.2" +name = "enum-iterator" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" dependencies = [ - "equator-macro", + "enum-iterator-derive", ] [[package]] -name = "equator-macro" -version = "0.4.2" +name = "enum-iterator-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ @@ -1308,6 +1547,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1324,6 +1574,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "exr" version = "1.74.2" @@ -1333,7 +1592,7 @@ dependencies = [ "bit_field", "half", "lebe", - "miniz_oxide", + "miniz_oxide 0.8.9", "num-complex", "pulp", "rayon-core", @@ -1341,6 +1600,16 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastdivide" version = "0.4.2" @@ -1349,18 +1618,18 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastembed" -version = "5.17.3" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c8600c9ec79b51d60c19911fe14eac04fe9c2895e87d2a3e80e2213d645a32" +checksum = "bf2876a78b0fa3a9095baf8bbec83126b3c5b7eab8fc4e1e7ad9e68b14ed5c8a" dependencies = [ - "anyhow", "hf-hub 0.5.0", "image", - "ndarray 0.17.2", + "ndarray", "ort", "safetensors", "serde", "serde_json", + "thiserror 2.0.20", "tokenizers", ] @@ -1395,29 +1664,43 @@ dependencies = [ ] [[package]] -name = "filetime" -version = "0.2.29" +name = "filedescriptor" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" dependencies = [ - "cfg-if", "libc", + "thiserror 1.0.69", + "winapi", ] [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "finl_unicode" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", + "zlib-rs", ] [[package]] @@ -1426,12 +1709,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -1474,12 +1751,12 @@ dependencies = [ [[package]] name = "fs4" -version = "0.8.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 0.38.44", - "windows-sys 0.52.0", + "rustix", + "windows-sys 0.59.0", ] [[package]] @@ -1499,9 +1776,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1514,9 +1791,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1524,15 +1801,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1541,38 +1818,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1585,6 +1862,12 @@ dependencies = [ "slab", ] +[[package]] +name = "gearhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616e8f476a586f1b078d9eece21f7a071f27997e709e2b471f5697deea53bda9" + [[package]] name = "generic-array" version = "0.14.7" @@ -1604,7 +1887,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1644,11 +1927,31 @@ dependencies = [ "weezl", ] +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "globset" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" dependencies = [ "aho-corasick", "bstr", @@ -1659,9 +1962,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1695,33 +1998,33 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.1.5", + "foldhash", + "serde", + "serde_core", ] [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.2.0", - "serde", - "serde_core", + "foldhash", ] [[package]] -name = "hashbrown" -version = "0.17.1" +name = "heapify" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" [[package]] name = "heck" @@ -1731,11 +2034,11 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" -version = "0.20.5" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" +checksum = "ad82d6598ccf1dac15c8b758a1bd282b755b6776be600429176757190a1b0202" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "heed-traits", "heed-types", @@ -1756,9 +2059,9 @@ checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" [[package]] name = "heed-types" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" dependencies = [ "bincode", "byteorder", @@ -1769,26 +2072,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] -name = "hf-hub" -version = "0.3.2" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" -dependencies = [ - "dirs 5.0.1", - "indicatif 0.17.11", - "log", - "native-tls", - "rand 0.8.7", - "serde", - "serde_json", - "thiserror 1.0.69", - "ureq 2.12.1", -] +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hf-hub" @@ -1806,11 +2098,65 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.19", - "ureq 3.3.0", + "thiserror 2.0.20", + "ureq", "windows-sys 0.61.2", ] +[[package]] +name = "hf-hub" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ccb6bcc85dec15413ef5949879f9a5497ca4568ed702547eb83fac23376e5c" +dependencies = [ + "base64 0.22.1", + "bon", + "bytes", + "futures", + "getrandom 0.2.17", + "globset", + "hf-xet", + "hyper", + "pathdiff", + "percent-encoding", + "reqwest 0.13.5", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.20", + "tokio", + "tokio-retry", + "tokio-util", + "tracing", + "url", + "wasm-bindgen-futures", +] + +[[package]] +name = "hf-xet" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c237ef4fb0ce1962a5117f8bd8c74454b41629826a9df17d14a1840ca18f0754" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "http", + "more-asserts", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -1825,9 +2171,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1845,9 +2191,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -1868,11 +2214,26 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -1972,9 +2333,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1986,9 +2347,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1999,9 +2360,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2013,16 +2374,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2033,15 +2395,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -2081,9 +2443,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.31" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8a7b8211e695a1d0cd91cace480d4d0bd57667ab10277cc412c5f7f4884f83" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" dependencies = [ "crossbeam-deque", "globset", @@ -2131,18 +2493,20 @@ dependencies = [ [[package]] name = "imgref" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" +checksum = "6e44b0a4eaa4c82f441d50a963f2d5f05a787240aeee097597033e72accfd22f" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -2154,7 +2518,7 @@ dependencies = [ "console 0.15.11", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width", "web-time", ] @@ -2164,9 +2528,9 @@ version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "console 0.16.4", + "console 0.16.6", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width", "unit-prefix", "web-time", ] @@ -2182,11 +2546,11 @@ dependencies = [ [[package]] name = "inotify" -version = "0.9.6" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.13.2", "inotify-sys", "libc", ] @@ -2202,27 +2566,15 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" +checksum = "2bf84e73fa6f27f299dec58e13223cf70db80da872eb921d4f6138342a0eabc8" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "indoc", "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "syn 3.0.5", ] [[package]] @@ -2237,21 +2589,19 @@ dependencies = [ ] [[package]] -name = "ipnet" -version = "2.12.0" +name = "inventory" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] [[package]] -name = "is-terminal" -version = "0.4.17" +name = "ipnet" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -2259,24 +2609,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2313,7 +2645,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -2362,20 +2694,48 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", "wasm-bindgen", ] +[[package]] +name = "kasuari" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" +dependencies = [ + "hashbrown 0.16.1", + "portable-atomic", + "thiserror 2.0.20", +] + +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + [[package]] name = "kqueue" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea" dependencies = [ "kqueue-sys", "libc", @@ -2387,10 +2747,16 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", ] +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + [[package]] name = "lazy_static" version = "1.5.0" @@ -2433,18 +2799,33 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" dependencies = [ "libc", ] [[package]] -name = "linux-raw-sys" -version = "0.4.15" +name = "line-clipping" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e752191d037c44ad111a8caa762921926658402f01cc1253f7bef2020ece4f5e" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" [[package]] name = "linux-raw-sys" @@ -2454,9 +2835,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "litrs" @@ -2486,9 +2867,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loop9" @@ -2501,24 +2882,36 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.5" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] -name = "lru-slab" -version = "0.1.2" +name = "lru" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] [[package]] name = "lzma-rust2" @@ -2526,21 +2919,31 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" +[[package]] +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix 0.29.0", + "winapi", +] + [[package]] name = "macro_rules_attribute" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" dependencies = [ "macro_rules_attribute-proc_macro", - "paste", + "pastey 0.2.3", ] [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" [[package]] name = "matchers" @@ -2553,9 +2956,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "matrixmultiply" @@ -2579,11 +2982,10 @@ dependencies = [ [[package]] name = "measure_time" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" dependencies = [ - "instant", "log", ] @@ -2602,12 +3004,37 @@ dependencies = [ "libc", ] +[[package]] +name = "memmem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2625,34 +3052,32 @@ dependencies = [ ] [[package]] -name = "mio" -version = "0.8.11" +name = "miniz_oxide" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.48.0", + "adler2", + "simd-adler32", ] [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] [[package]] name = "moka" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -2687,6 +3112,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "moxcms" version = "0.8.1" @@ -2697,6 +3128,33 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multiversion" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 3.0.5", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "murmurhash32" version = "0.3.1" @@ -2720,21 +3178,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - [[package]] name = "ndarray" version = "0.17.2" @@ -2756,13 +3199,26 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nix" version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -2810,37 +3266,44 @@ checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" [[package]] name = "notify" -version = "6.1.1" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.13.1", - "crossbeam-channel", - "filetime", + "bitflags 2.13.2", "fsevent-sys", "inotify", "kqueue", "libc", "log", - "mio 0.8.11", + "mio", + "notify-types", "walkdir", - "windows-sys 0.48.0", + "windows-sys 0.60.2", ] [[package]] name = "notify-debouncer-full" -version = "0.3.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fd166739789c9ff169e654dc1501373db9d80a4c3f972817c8a4d7cf8f34e" +checksum = "c02b49179cfebc9932238d04d6079912d26de0379328872846118a0fa0dbb302" dependencies = [ - "crossbeam-channel", "file-id", "log", "notify", - "parking_lot", + "notify-types", "walkdir", ] +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -2898,9 +3361,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2923,7 +3386,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -2936,6 +3398,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -2957,7 +3428,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", + "dispatch2", + "objc2", ] [[package]] @@ -2966,6 +3439,16 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + [[package]] name = "objc2-io-kit" version = "0.3.2" @@ -2976,6 +3459,26 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3000,7 +3503,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -3028,7 +3531,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -3080,35 +3583,53 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" +dependencies = [ + "num-traits", +] + [[package]] name = "ort" -version = "2.0.0-rc.12" +version = "2.0.0-rc.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" dependencies = [ - "ndarray 0.17.2", + "ndarray", "ort-sys", "smallvec", "tracing", - "ureq 3.3.0", + "ureq", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.12" +version = "2.0.0-rc.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" dependencies = [ "hmac-sha256", "lzma-rust2", - "ureq 3.3.0", + "ureq", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", ] [[package]] name = "ownedbytes" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" dependencies = [ "stable_deref_trait", ] @@ -3123,6 +3644,39 @@ dependencies = [ "winapi", ] +[[package]] +name = "palette" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddeed8580d347d2abf3dcf06a5f0b3dc020258338526b277847cd4248a70fc64" +dependencies = [ + "approx", + "libm", + "palette_derive", + "palette_math", +] + +[[package]] +name = "palette_derive" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88537020289b719d81be994ccf1bbf4990f477e2f69ee52fe3e45f43a02e56be" +dependencies = [ + "by_address", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "palette_math" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e6eb142958d64335fb0e345c5b9ead2ecd6fc438c307e9d7d3c4fd428dbaf12" +dependencies = [ + "libm", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3164,6 +3718,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -3179,6 +3739,48 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +dependencies = [ + "pest", +] + [[package]] name = "phf" version = "0.11.3" @@ -3189,6 +3791,16 @@ dependencies = [ "phf_shared", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + [[package]] name = "phf_generator" version = "0.11.3" @@ -3196,7 +3808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -3221,6 +3833,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3229,9 +3861,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "plotters" @@ -3267,33 +3899,33 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "crc32fast", "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.8.9", ] [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -3323,6 +3955,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.5", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -3417,19 +4059,19 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3437,9 +4079,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "aws-lc-rs", "bytes", @@ -3448,11 +4090,11 @@ dependencies = [ "rand 0.10.2", "rand_pcg", "ring", - "rustc-hash 2.1.3", + "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3495,12 +4137,10 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ - "libc", - "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -3510,7 +4150,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3525,16 +4165,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3550,9 +4180,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_core" @@ -3570,43 +4197,113 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] -name = "rand_distr" -version = "0.4.3" +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "num-traits", - "rand 0.8.7", + "rand_core 0.10.1", ] [[package]] -name = "rand_pcg" -version = "0.10.2" +name = "ratatui" +version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +checksum = "3274ba0a2c5e1bcad2a2005d20f4dc59dad26b2eb0940fb094500dba4099d57d" dependencies = [ - "rand_core 0.10.1", + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-macros", + "ratatui-termina", + "ratatui-termwiz", + "ratatui-widgets", + "serde", ] [[package]] -name = "ratatui" -version = "0.29.0" +name = "ratatui-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" +dependencies = [ + "bitflags 2.13.2", + "compact_str", + "critical-section", + "hashbrown 0.17.1", + "itertools 0.14.0", + "kasuari", + "lru 0.18.4", + "palette", + "serde", + "strum", + "thiserror 2.0.20", + "unicode-segmentation", + "unicode-truncate", + "unicode-width", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" dependencies = [ - "bitflags 2.13.1", - "cassowary", - "compact_str 0.8.2", + "cfg-if", "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-macros" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7dc68daa7498a43e4d68e0eb078427e10c38fbcfbb1e42d955f1fa2140d814" +dependencies = [ + "ratatui-core", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-termina" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0bf912d9e66f057a759d92e386a280ea886b352ab757d6ac4d653c7ed2c43c2" +dependencies = [ + "instability", + "ratatui-core", + "termina", +] + +[[package]] +name = "ratatui-termwiz" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf03e0380b7744054d6cb74224fe3adf062a029754933f575ca1e3b4c2ce977" +dependencies = [ + "ratatui-core", + "termwiz", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" +dependencies = [ + "bitflags 2.13.2", + "hashbrown 0.17.1", "indoc", "instability", - "itertools 0.13.0", - "lru", - "paste", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "serde", "strum", + "time", "unicode-segmentation", - "unicode-truncate", - "unicode-width 0.2.0", + "unicode-width", ] [[package]] @@ -3637,9 +4334,9 @@ dependencies = [ "paste", "profiling", "rand 0.9.5", - "rand_chacha 0.9.0", + "rand_chacha", "simd_helpers", - "thiserror 2.0.19", + "thiserror 2.0.20", "v_frame", "wasm-bindgen", ] @@ -3665,7 +4362,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -3712,23 +4409,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" [[package]] -name = "redox_syscall" -version = "0.5.18" +name = "redb" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" dependencies = [ - "bitflags 2.13.1", + "libc", ] [[package]] -name = "redox_users" -version = "0.4.6" +name = "redox_syscall" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", + "bitflags 2.13.2", ] [[package]] @@ -3739,27 +4434,27 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -3776,9 +4471,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -3836,14 +4531,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -3852,6 +4549,8 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", + "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -3860,6 +4559,7 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", @@ -3874,6 +4574,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest 0.13.5", + "thiserror 2.0.20", + "tower-service", +] + [[package]] name = "rgb" version = "0.8.53" @@ -3896,28 +4610,29 @@ dependencies = [ [[package]] name = "rmcp" -version = "1.8.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1f571c72940a19d9532fe52dbea8bc9912bf1d766c2970bb824056b86f3f59" +checksum = "b88db56b8ae316560e9e868b6b978ea940f27cb883323fc90e07435e44158f5c" dependencies = [ "async-trait", - "base64 0.22.1", + "base64 0.23.1", "bytes", "chrono", "futures", "http", "http-body", "http-body-util", + "indexmap", "pastey 0.2.3", "pin-project-lite", "rand 0.10.2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp-macros", "schemars", "serde", "serde_json", "sse-stream", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -3928,22 +4643,22 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "1.8.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aad0035b69380782d78ea95b508327e6deaa2235909053e596eea8f27b5e1d5" +checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", "serde_json", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "roaring" -version = "0.10.12" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" dependencies = [ "bytemuck", "byteorder", @@ -3959,12 +4674,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.3" @@ -3980,37 +4689,24 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -4036,9 +4732,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -4073,9 +4769,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -4095,6 +4791,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + [[package]] name = "safetensors" version = "0.8.0" @@ -4128,9 +4830,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "chrono", "dyn-clone", @@ -4142,21 +4844,21 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "scip" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a72133c2d6fd45c9a3a343bcb3db2faa30f68f0919bfa3370ca85add5460c3" +checksum = "d7f651fbd0f98742a47b58623a7bcfd8fad14455912689e4867a1f43ee7e02c6" dependencies = [ "protobuf", ] @@ -4173,7 +4875,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4223,18 +4925,18 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] name = "serde_derive_internals" -version = "0.29.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -4262,6 +4964,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4274,6 +4987,31 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sha2" version = "0.10.9" @@ -4282,7 +5020,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4294,6 +5043,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs 6.0.0", + "os_str_bytes", +] + [[package]] name = "shlex" version = "2.0.1" @@ -4317,7 +5077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", - "mio 1.2.2", + "mio", "signal-hook", ] @@ -4370,9 +5130,9 @@ checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "sketches-ddsketch" -version = "0.2.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" dependencies = [ "serde", ] @@ -4385,9 +5145,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" [[package]] name = "socket2" @@ -4424,9 +5184,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4" dependencies = [ "bytes", "futures-util", @@ -4447,6 +5207,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "streaming-iterator" version = "0.1.9" @@ -4461,23 +5231,22 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", "quote", - "rustversion", "syn 2.0.119", ] @@ -4493,6 +5262,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.119" @@ -4506,9 +5286,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -4558,13 +5338,28 @@ dependencies = [ "windows", ] +[[package]] +name = "sysinfo" +version = "0.39.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "objc2-open-directory", + "windows", +] + [[package]] name = "system-configuration" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4587,37 +5382,38 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.22.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +checksum = "861facfabd71044968f364837f9a083b56464ba5a59079f88706ee5c451ca069" dependencies = [ "aho-corasick", "arc-swap", "base64 0.22.1", "bitpacking", + "bon", "byteorder", "census", "crc32fast", "crossbeam-channel", + "datasketches", "downcast-rs", "fastdivide", "fnv", "fs4", "htmlescape", - "itertools 0.12.1", + "itertools 0.14.0", "levenshtein_automata", "log", - "lru", + "lru 0.16.4", "lz4_flex", "measure_time", "memmap2", - "num_cpus", "once_cell", "oneshot", "rayon", "regex", "rust-stemmers", - "rustc-hash 1.1.0", + "rustc-hash", "serde", "serde_json", "sketches-ddsketch", @@ -4630,30 +5426,31 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.20", "time", + "typetag", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.6.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools 0.12.1", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4663,9 +5460,9 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.7.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" dependencies = [ "async-trait", "byteorder", @@ -4687,19 +5484,25 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.22.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" dependencies = [ + "fnv", "nom 7.1.3", + "ordered-float 5.5.0", + "serde", + "serde_json", ] [[package]] name = "tantivy-sstable" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ + "futures-util", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -4708,20 +5511,19 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" dependencies = [ "murmurhash32", - "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" dependencies = [ "serde", ] @@ -4735,10 +5537,86 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] +[[package]] +name = "termina" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9048a889effe34a5cddee0af7f53285198b16dca3be510858d38dfdb3e62a04e" +dependencies = [ + "bitflags 2.13.2", + "parking_lot", + "rustix", + "signal-hook", + "windows-sys 0.61.2", +] + +[[package]] +name = "terminfo" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" +dependencies = [ + "fnv", + "nom 7.1.3", + "phf", + "phf_codegen", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + +[[package]] +name = "termwiz" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bitflags 2.13.2", + "fancy-regex", + "filedescriptor", + "finl_unicode", + "fixedbitset", + "hex", + "lazy_static", + "libc", + "log", + "memmem", + "nix 0.29.0", + "num-derive", + "num-traits", + "ordered-float 4.6.0", + "pest", + "pest_derive", + "phf", + "sha2 0.10.9", + "signal-hook", + "siphasher", + "terminfo", + "termios", + "thiserror 1.0.69", + "ucd-trie", + "unicode-segmentation", + "vtparse", + "wezterm-bidi", + "wezterm-blob-leases", + "wezterm-color-types", + "wezterm-dynamic", + "wezterm-input-types", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4750,11 +5628,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -4770,13 +5648,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -4804,12 +5682,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", + "libc", "num-conv", + "num_threads", "powerfmt", "serde_core", "time-core", @@ -4834,9 +5714,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -4854,28 +5734,19 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokenizers" -version = "0.22.2" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" dependencies = [ "ahash", - "aho-corasick", - "compact_str 0.9.1", + "compact_str", + "daachorse", "dary_heap", "derive_builder", "esaxx-rs", @@ -4894,7 +5765,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 2.0.19", + "thiserror 2.0.20", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -4908,7 +5779,7 @@ checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", - "mio 1.2.2", + "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -4919,13 +5790,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -4938,11 +5809,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-retry" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" +dependencies = [ + "pin-project-lite", + "rand 0.10.2", + "tokio", +] + [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -4974,6 +5856,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "tower" version = "0.5.3" @@ -4996,7 +5902,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -5005,7 +5911,6 @@ dependencies = [ "tower", "tower-layer", "tower-service", - "tracing", "url", ] @@ -5041,7 +5946,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "tracing-subscriber", ] @@ -5111,13 +6016,12 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.26.11" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1c71c1c4cc0920b20d6b0f6572e7682cd07a6a2faec71067a31fa394c586df" +checksum = "2038684e0058edba0d17302619f62eabce4a8e11c6ac59506996a8d79848851d" dependencies = [ "cc", "regex", - "regex-syntax", "serde_json", "streaming-iterator", "tree-sitter-language", @@ -5215,9 +6119,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" [[package]] name = "tree-sitter-md" @@ -5241,9 +6145,9 @@ dependencies = [ [[package]] name = "tree-sitter-proto" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e410ccb5fa3cbd6bf7b8e512ecf7ad9d5254395b822bfe9f751b50fa978f31c" +checksum = "9c93b6f1ed20de442e900eb636f2176af6063953dfaa9f76bf168dd3b490a3a1" dependencies = [ "cc", "tree-sitter-language", @@ -5305,12 +6209,66 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -5334,21 +6292,15 @@ checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-truncate" -version = "1.1.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "unicode-segmentation", - "unicode-width 0.1.14", + "unicode-width", ] -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.0" @@ -5375,30 +6327,11 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.12.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" dependencies = [ - "base64 0.22.1", - "flate2", - "log", - "native-tls", - "once_cell", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "url", - "webpki-roots 0.26.11", -] - -[[package]] -name = "ureq" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" -dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "cookie_store", "der", "flate2", @@ -5413,16 +6346,16 @@ dependencies = [ "ureq-proto", "utf8-zero", "webpki-root-certs", - "webpki-roots 1.0.9", + "webpki-roots", ] [[package]] name = "ureq-proto" -version = "0.6.0" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +checksum = "5b0809a01d1ca5a51ca70db32bb2a19157582a526505ef3c19e3b343a59aa5ad" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "http", "httparse", "log", @@ -5440,6 +6373,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -5466,10 +6405,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ + "atomic", "getrandom 0.4.3", "js-sys", "serde_core", @@ -5505,6 +6445,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vtparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9b2acfb050df409c972a37d3b8e08cdea3bddb0c09db9d53137e504cfabed0" +dependencies = [ + "utf8parse", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5530,6 +6479,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -5539,11 +6497,20 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -5554,9 +6521,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -5564,9 +6531,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5574,22 +6541,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -5622,9 +6589,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -5649,15 +6616,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.9", -] - [[package]] name = "webpki-roots" version = "1.0.9" @@ -5673,6 +6631,91 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "wezterm-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" +dependencies = [ + "log", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-blob-leases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" +dependencies = [ + "getrandom 0.3.4", + "mac_address", + "sha2 0.10.9", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "wezterm-color-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de81ef35c9010270d63772bebef2f2d6d1f2d20a983d27505ac850b8c4b4296" +dependencies = [ + "csscolorparser", + "deltae", + "lazy_static", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" +dependencies = [ + "log", + "ordered-float 4.6.0", + "strsim", + "thiserror 1.0.69", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c0cf2d539c645b448eaffec9ec494b8b19bd5077d9e58cb1ae7efece8d575b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wezterm-input-types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" +dependencies = [ + "bitflags 1.3.2", + "euclid", + "lazy_static", + "serde", + "wezterm-dynamic", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5816,15 +6859,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -5861,21 +6895,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -5918,12 +6937,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5936,12 +6949,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5954,12 +6961,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5984,12 +6985,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6002,12 +6997,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6020,12 +7009,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6038,12 +7021,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -6064,9 +7041,150 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xet-client" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b8da8cc70aa2e3c500c0400e012df82c656ab9fca47f9f939fffc5afd89aca" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "more-asserts", + "rand 0.10.2", + "redb", + "reqwest 0.13.5", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-retry", + "tokio_with_wasm", + "tracing", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "73503c223783dccc864abde22115e09d12f190448a0baf58ab2c54bc709e2f99" +dependencies = [ + "async-trait", + "base64 0.22.1", + "blake3", + "bytemuck", + "bytes", + "countio", + "futures", + "futures-util", + "getrandom 0.4.3", + "heapify", + "itertools 0.14.0", + "lz4_flex", + "more-asserts", + "rand 0.10.2", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89052ec5dec2187cad30b86af92cc24fd61c4a57a795f1ff7ff5f38d49184eb" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "gearhash", + "http", + "itertools 0.14.0", + "more-asserts", + "rand 0.10.2", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "url", + "uuid", + "web-time", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af5c60d5eed38ab4c576f4421bae835e7bd07631fb381705605529d2015c106b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored 3.1.1", + "const-str", + "ctor", + "dirs 6.0.0", + "futures", + "git-version", + "humantime", + "konst", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.2", + "reqwest 0.13.5", + "serde", + "serde_json", + "shellexpand", + "sysinfo 0.38.4", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "tracing-appender", + "tracing-subscriber", + "web-time", + "whoami", + "winapi", +] [[package]] name = "y4m" @@ -6105,18 +7223,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -6152,9 +7270,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -6163,9 +7281,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -6174,15 +7292,21 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" @@ -6200,18 +7324,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", @@ -6219,9 +7343,9 @@ dependencies = [ [[package]] name = "zune-core" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" [[package]] name = "zune-inflate" diff --git a/Cargo.toml b/Cargo.toml index f084c04d..f5f24e18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.2.10" +version = "1.4.2" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" @@ -25,19 +25,19 @@ tokio = { version = "1.40", features = ["full"] } tokio-util = { version = "0.7", features = ["rt"] } ctrlc = "3.4" anyhow = "1.0" -thiserror = "1.0" +thiserror = "2.0" # ML & Embeddings -fastembed = "5.0" +fastembed = "6.1" ort = { version = "2.0.0-rc.10", features = ["download-binaries", "copy-dylibs"] } -ndarray = "0.16" -hf-hub = "0.3" +ndarray = "0.17" +hf-hub = "1.0" # Vector DB (temporarily commented due to dep conflicts) # lancedb = "0.5" # Text processing & Parsing -tree-sitter = "0.26.8" +tree-sitter = "0.27" tree-sitter-rust = "0.24.2" tree-sitter-python = "0.25" tree-sitter-javascript = "0.25" @@ -54,12 +54,12 @@ tree-sitter-yaml = "0.7.2" tree-sitter-json = "0.24.8" tree-sitter-md = "0.5.3" tree-sitter-dart = "0.2.0" -tree-sitter-proto = "0.4.0" +tree-sitter-proto = "0.6" # File handling ignore = "0.4" -notify = { version = "6.1", default-features = false, features = ["macos_fsevent"] } -notify-debouncer-full = "0.3" +notify = { version = "8.2", default-features = false, features = ["macos_fsevent"] } +notify-debouncer-full = "0.7" walkdir = "2.5" fs2 = "0.4" # Cross-platform file locking @@ -67,17 +67,15 @@ fs2 = "0.4" # Cross-platform file locking moka = { version = "0.12", features = ["sync"] } # Search & Ranking -tantivy = "0.22" +tantivy = "0.26" regex = "1.12" # Server -axum = "0.7" -tower = "0.5" -tower-http = { version = "0.6", features = ["cors", "trace"] } +axum = "0.8" # TUI -ratatui = "0.29" -crossterm = "0.28" +ratatui = "0.30" +crossterm = "0.29" # Utilities rayon = "1.10" @@ -87,37 +85,39 @@ serde_json = "1.0" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } tracing-appender = "0.2" -sha2 = "0.10" +sha2 = "0.11" uuid = { version = "1.11", features = ["v4", "serde"] } chrono = { version = "0.4", features = ["serde"] } colored = "2.1" indicatif = "0.17" -dirs = "5.0" +dirs = "7" num_cpus = "1.16" async-trait = "0.1" # Vector database -arroy = "0.5" -heed = "0.20" +arroy = "0.8" +heed = "0.22" bincode = "1.3" # SCIP symbol indexing — parses standard SCIP protobuf (.scip) emitted by # Sourcegraph indexers (e.g. scip-typescript) for the TypeScript symbol adapter. -scip = "0.9" +scip = "0.10" protobuf = "3.7" -rand = "0.8" -# Bumped floor from 1.5.0 to 1.8.0 for CVE patches (Aikido group: rmcp priority 82, 3 CVEs). -# v2.x is available but is a breaking major bump — deferred. -rmcp = { version = "1.8.0", features = ["server", "client", "transport-io", "transport-streamable-http-server", "transport-streamable-http-client-reqwest", "macros"] } +rand = "0.10" +# Aikido 34247111: 1.8.0 carries 5 advisories (CVE-2026-64684/317735/617985 + 2 more). +# 3.x keeps legacy initialize/session behavior by default (ProtocolVersion::LATEST stays +# V_2025_11_25); the 2026-07-28 stateless lifecycle is opt-in and NOT enabled here. +rmcp = { version = "3.3.0", features = ["server", "client", "transport-io", "transport-streamable-http-server", "transport-streamable-http-client-reqwest", "macros"] } schemars = { version = "1.1.0", features = ["derive"] } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } -sysinfo = { version = "0.38.4", default-features = true } +sysinfo = { version = "0.39", default-features = true } [target.'cfg(unix)'.dependencies] libc = "0.2" [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.8", features = ["html_reports"] } tempfile = "3.13" pretty_assertions = "1.4" +serial_test = "3" # Benchmarks will be added later # [[bench]] diff --git a/DIAGNOSE_FEDERATED_KEEP_WARM.md b/DIAGNOSE_FEDERATED_KEEP_WARM.md deleted file mode 100644 index 609d0b2b..00000000 --- a/DIAGNOSE_FEDERATED_KEEP_WARM.md +++ /dev/null @@ -1,220 +0,0 @@ -# Diagnosis — a federated cloud peer waking up with nobody querying it - -_Branch: `fix/federated-silent-poll-diagnosis` — 2026-08-05_ - -> **Status: root cause CONFIRMED against Azure Log Analytics ground truth.** -> An earlier revision of this document blamed a misconfigured local -> `CODESEARCH_KEEP_WARM_URL`. That hypothesis was **disproven** — see -> [What was ruled out](#what-was-ruled-out) §4. The confirmed cause is a -> two-part defect described in [Root cause](#root-cause). The corresponding -> fixes are listed in [Fixes](#fixes). - -## The requirement being violated - -Background polling of **local** repos is fine and expected. Background -polling of a **federated peer** must never happen — it was an explicit -design constraint from the original federation design, restated by the -reporting user as: - -> "hij mag die repos pollen LOKAAL maar niet federated !!! dat had ik -> nochthans in de specs effectief gezegd bij het ontwerp" - -Two things follow, and conflating them is what caused three round-trips on -this same behaviour: - -- A peer staying warm for its full idle window **after real use** is - *correct*. That is what keep-warm is for. -- A peer being **woken** with no federated query behind it is the defect — - as is it then staying warm for an hour off that spurious wake. - -"Cannot keep a peer awake past the host's own suspend term" is a strictly -**weaker** property than "never wakes it", and only the latter was ever the -requirement. - -## Symptom reported - -A local `codesearch serve` instance kept a mounted cloud federation peer (an -Azure Container Apps replica, `minReplicas: 0`) alive. Quitting the local -instance stopped it. The peer would wake, stay up ~1 hour, sleep, and wake -again — with no federated searches performed in between. Nothing appeared in -the local logs for any of it. - -## Ground truth - -From Log Analytics (`ContainerAppSystemLogs_CL` / `ContainerAppConsoleLogs_CL`) -on the deployed peer, over a period with **zero** federated searches: - -| Observation | Value | -|---|---| -| Interval between wakes | **120, 121, 120 minutes** | -| Warm period per wake | **~67 min** (1h idle window + 5min KEDA `cooldownPeriod`) | -| Nightly sleeps | exactly **2h00m30s** apart | -| Resulting duty cycle | **≈13.4h warm/day, ~56%** — at zero searches | - -The 120-minute spacing is the tell: it is the **local** host's -`DEFAULT_IDLE_SUSPEND_SECS` (2h), not any value configured on the peer. - -Each wake additionally paid for an `azcopy sync` of the docs blob and a -`git pull` of the KB repo. - -## Root cause - -Two independent defects, one triggering and one amplifying. - -### Defect 1 — the trigger: the TUI polled federated peers on a timer - -`spawn_remote_discovery` in `src/serve/tui.rs` used -`Duration::from_secs(state.idle_suspend_secs())` as a baseline poll interval -and, on each elapse, ran a `JoinSet` `/status` fan-out to **every** -configured peer. On the local host that value is 2h — matching the observed -cadence exactly. - -Each fan-out woke the peer's scale-to-zero replica. Nothing else was needed: -the poll *itself* was the ingress traffic. - -The reasoning that shipped this — recorded here so it is not reintroduced a -fourth time — was that polling no faster than the host's own suspend term is -harmless. It is not, for two separate reasons: - -1. Not keeping a peer awake *past* its suspend term says nothing about not - *waking* it. The peer's warm time is bounded, but its wake **count** is - not zero, and each wake costs a full warm window. -2. The two windows are unrelated values. `idle_suspend_secs` was read from - the **local** process (2h default); the window the woken peer then - honoured was the **peer's** (~1h). PR #181's description claimed the - cadence was "1h on the cloud deploy" — it was reading the local value. - -### Defect 2 — the amplifier: keep-warm rewarded spurious wakes - -The cloud keep-warm loop in `src/serve/mod.rs` computed its idle check as: - -```rust -let last = kw_state.most_recent_tool_call().unwrap_or(start); -``` - -`/status` and `/healthz` do **not** call `record_tool_call`. So a replica -woken by anything other than a genuine tool call found no recorded tool -call, fell back to the process start time, and self-pinged its own ingress -every `KEEP_WARM_INTERVAL_SECS` (120s) for the entire idle window. - -The critical observation is that this fallback is **unreachable in the case -it was written for**: a real tool call always sets `last_tool_call`, so the -`unwrap_or` only ever fires when the wake was *not* real work. Its whole -practical effect was to convert a momentary spurious wake into a full warm -hour — roughly **11× amplification** (~67 min instead of the ~6 min a bare -wake would have cost). - -### How they combine - -Defect 1 wakes the peer every 2h. Defect 2 then holds it up for ~67 min per -wake. Neither alone produces the observed 56% duty cycle; together they do. - -## What was ruled out - -1. **Explicit federated tool calls** (`federated_search`, - `federated_project_search`, `federated_get_chunk` in `src/mcp/mod.rs`) — - the only callers of `record_remote_peer_activity`, and only reached when a - project resolves to a federated alias. No federation-shaped log lines - existed in a full day's logs for either the reporting instance or an - unrelated local hub used to cross-check. -2. **`Watch-CodesearchServeReplicas.ps1`** — does poll `/status` every 20s, - but last ran 2026-07-05, well before the observed window. -3. **A stale binary re-introducing an old bug** — the reported startup banner - was `v1.2.1`. Worth upgrading, but the 2h cadence exists in that version - too. -4. **A misconfigured local `CODESEARCH_KEEP_WARM_URL`** *(the earlier - revision's stated root cause — disproven)*, on four independent grounds: - - The env var is set **nowhere** locally: not in the process environment, - not in `HKCU`, not in `HKLM`, not in any shell profile. - - The one-time `🔥 keep-warm enabled` line appears in **zero** local logs - from 2026-04-26 onward. - - That absence is meaningful: `init_serve_logger` is *always* file-only in - serve mode, unconditional on `--no-tui`, and those logs do carry other - `INFO` lines — so the line would have been captured had it fired. - - No local `codesearch` process held any connection on `:443`. - -Note that the earlier revision also ruled out TUI federated polling, on the -grounds that `maybe_spawn_tui` is gated on `!no_tui && is_tty()`. That gating -is real, but the conclusion was wrong: the reporting user's *waking* instance -was a normal TTY serve with the TUI running. Only the separate `--no-tui` -cross-check instance was exempt. - -## Fixes - -### Shipped earlier on this branch (commit `55fa36b`) - -Keep-warm observability, in `src/serve/mod.rs`: - -1. **Per-ping logging** — success at `debug!`, failure at `warn!`. Previously - `let _ = client.get(&ping_url)...send().await;` discarded both, leaving a - single one-time "enabled" line as the feature's only trace. -2. **Startup misconfiguration warning** — `extract_host_from_url` (no new - dependency) compares the keep-warm target host against the server's own - bind host and warns when they differ. - -Tests: `src/serve/tests.rs::keep_warm_host_extraction_tests`. - -### Defect 1 — no timer poll of federated peers - -`spawn_remote_discovery` no longer polls on any cadence. The periodic tick is -**config-only** (`REMOTE_ROW_REFRESH_SECS` = 5s, zero HTTP): it rebuilds -mounted-remote rows from the `remote_mounts` allowlist so mount/unmount edits -and `l` reloads surface promptly, and contacts nobody. - -A peer is contacted only by: - -- an **activity poke** — a real federated tool call just landed on that peer, - so it is demonstrably already awake; only that peer is refreshed, never a - fan-out, so an idle sibling peer is untouched; -- the explicit **`i`** info-overlay keypress on a remote row. - -Consequences: an idle mount renders its activity as `-`, which is now the -correct steady state rather than a fault. `ServeState::idle_suspend_secs` -(field, env init, getter and `--idle-suspend-secs` override) is removed — it -existed only to feed the poll cadence and became write-only. The keep-warm -task resolves flag > env > default directly, so `--idle-suspend-secs` is -unchanged. The `initial_cycle` startup gate is gone: every cycle is now -config-only, so it had nothing left to gate. - -Also fixed in passing: the snapshot emit was gated on a non-empty peer list, -so removing the *last* peer from `repos.json` left its rows on screen -forever. It is now unconditional. - -### Defect 2 — keep-warm requires a real tool call - -The `unwrap_or(start)` fallback is removed: with no tool call recorded there -is nothing to keep warm for, so the loop simply does not ping. A freshly -deployed replica now sleeps until first real use instead of self-warming for -an hour, which is the intended behaviour of scale-to-zero. - -### Follow-up — the `55fa36b` warning false-positived on the correct deploy - -The startup "target isn't self" warning fired on the **only deployment where -keep-warm is correct**: on Azure the process binds `0.0.0.0` while -`keep_warm_url` is the ingress FQDN, so `looks_like_self` was false and the -warning fired on every cold start. A wildcard bind means the -externally-visible host is genuinely unknown, so the comparison cannot -conclude anything and must stay silent — a check that cries wolf on the -correct configuration trains operators to ignore the case that matters. - -Fixed alongside Defect 2. The rule now lives in a testable -`keep_warm_foreign_target(ping_url, self_host) -> Option` helper -(`None` = do not warn), covered by tests for wildcard binds, a genuine -foreign host, a matching host, loopback targets, and an unparseable URL. - -## Residual surface (known, not currently exploitable) - -The MCP **`status` tool** passes `allow_unscoped = true`, but when it is -*project-scoped* (or the replica is single-repo) `is_multi` is false, so the -`!allow_unscoped || !is_multi` guard lets it through and it **does** record a -tool call. An automated poller calling the MCP `status` *tool* with -`project=` would therefore still buy a full warm window. - -No such poller is known to exist: both `Watch-CodesearchServeReplicas.ps1` -and `FederationClient::list_repos` use the **HTTP** `/status` endpoint -(`status_handler`), which does not record. Noted here so that if the -symptom ever recurs, this is the first place to look. - -## Local repos - -Unaffected by all of the above, by design. diff --git a/DIAGNOSE_FIND_IMPACT_ROUTING.md b/DIAGNOSE_FIND_IMPACT_ROUTING.md deleted file mode 100644 index 2f912aef..00000000 --- a/DIAGNOSE_FIND_IMPACT_ROUTING.md +++ /dev/null @@ -1,271 +0,0 @@ -# DIAGNOSE — Waarom kiest de agent zelden `find_impact`? - -> **Status:** DIAGNOSE-EERST. Dit document levert geen fix, maar een reproduceerbare -> analyse met gehard bewijs uit de broncode, een hypotheses-overzicht, een geïsoleerde -> oorzaak, en pas daarna gefaseerde fix-opties (geen blinde oplossing). -> **Symptoom:** de agent pakt voor "wie roept X aan / wat breekt als ik X hernoem" -> vrijwel altijd `find kind=usages` (BM25/tekst-benadering) of `search(semantic)`, -> zelden `find_impact` — terwijl `find_impact` het enige SCIP-backed call-graph-pad is. -> **Repo:** `codesearch-git`. Validatie: `cargo check` + `cargo clippy -D warnings`. - ---- - -## 1. Doel & scope - -**In scope** -- Vaststellen **waarom** de agent `find_impact` mijdt, met bewijs op 3 lagen: - server-instructies, tool-descriptions, en deploy-realiteit. -- De keuze instrumenteerbaar maken (zowel server- als agent-kant). -- Gefundeerde fix-opties aandragen — niet blind één implementeren. - -**Niet in scope (pas ná isolatie)** -- De daadwerkelijke code-fix. Die volgt uit de gekozen optie in §7. -- TS/andere-talen SCIP-backends (apart plan: `PLAN_TYPESCRIPT_SCIP.md`). - ---- - -## 2. Symptoom & observatie - -| Vraagtype | Verwachte tool | Werkelijk gekozen (observatie) | -|-----------|----------------|--------------------------------| -| "wie roept `foo()` aan?" | `find_impact` | `find kind=usages` of `search` | -| "wat breekt als ik `Bar` hernoem?" | `find_impact` | `find kind=usages` | -| "toon call-graph van `X`" | `find_impact` | `search(semantic)` of `find` | - -Het gedrag is **consistent reproduceerbaar**: stel de vraag in een willekeurige -agent-sessie die codesearch-MCP gebruikt → agent kiest `find`/`search`, niet `find_impact`. - ---- - -## 3. Bewijsmateriaal uit de broncode (hard evidence) - -De oorzaak is niet verborgen — ze staat letterlijk in wat de server aan de agent -voert. Drie lagen, allemaal in `src/mcp/mod.rs`: - -### 3.1 Server-instructies (worden in de agent system-prompt geïnjecteerd) -`INSTRUCTIONS_TEMPLATE` (`src/mcp/mod.rs:7915-7953`) — exacte regels die de agent ziet: - -``` -PICK THE RIGHT TOOL FOR THE TASK: - "who calls X?" / "what breaks if I rename X?" - → find_impact (C# via SCIP; other languages: use find kind="usages") ← 7931 -RULES: - - search(semantic) is the DEFAULT for code lookup. Don't skip it. ← 7944 - - find_impact for C# refactors; find(kind="usages") for other languages. ← 7945 -``` - -**Drie biases in deze tekst:** -1. Regel 7931 routeert "who calls X?" voor **elke niet-C# taal** expliciet naar `find kind=usages`. -2. Regel 7944 positioneert `search(semantic)` als de **DEFAULT** — alles wat niet expliciet anders is, valt terug op search. -3. Regel 7945 kadermt `find_impact` als "C# **refactors**" — smal, niet als algemene call-graph-tool. - -### 3.2 `find_impact` tool-description (`src/mcp/mod.rs:6236`) -``` -"Symbol impact analysis — find all references ... (SCIP). - ... More accurate than text-based `find kind=\"usages\"` ... - Languages: C# today (requires the `scip-csharp` helper ...). - For Rust/Python/Go/etc., use `find` with `kind=\"usages\"` as a text-based fallback - until SCIP backends for those languages ship." ← ACTIEVE DOORVERWIJZING WEG -``` -De tool-description **zelf** zegt de agent om `find_impact` te vermijden voor niet-C#. -Dit is de sterkste bias: de tool die we willen promoten, ontmoedigt zichzelf. - -### 3.3 `find` tool-description (`src/mcp/mod.rs:4611`) -``` -"- `usages`: find all call-sites and references to a symbol" -``` -Generiek, geen caveat, geen verwijzing dat `find_impact` preciezer is. `find` presenteert -zich als het algemene antwoord op "who calls X" — voor **alle** talen, zonder drempel. - -### 3.4 README + zoekresultaat-meta (versterking) -- `README.md:307-321`: publieke docs framen `find_impact` als "Currently supports **C#**", - "Requires the `-with-csharp` release variant". -- **Ironische meta-observatie:** de server emit bij zwakke zoekresultaten zelf een - `suggested_tool: "find with kind=usages"` note — dus het systeem adviseert actief `find`, - nooit `find_impact`. - -### 3.5 Deploy-realiteit (de derde laag) -`find_impact` faalt als er geen `scip-csharp` helper is (`mcp/mod.rs:6332-6347`, -`is_available()` check → retourneert een error-JSON met `hint_for_agent`). Op een -serve-hub **zonder** `-with-csharp` variant faalt `find_impact` dus altijd. Een agent -die het één keer probeert en een error terugkrijgt, leert het daarna vermijden — -self-reinforcing. `find kind=usages` faalt nooit (puur tekst-index, altijd aan). - ---- - -## 4. (a) Reproduceren & instrumenteren - -Doel: **meetbaar** maken welke tool de agent kiest en waarom, bij welke queries. - -### 4.1 Wat de server al logt (server-kant = "welke tool") -`tracing::info!` bij elk tool-call: -- `find_impact`: `mcp/mod.rs:6242` (symbol_name, file, line, language, project) -- `find`: `mcp/mod.rs:4622` (symbol, kind, project, group) -- `search`: aparte `📥 search` log - -→ **De "welke tool" is al traceerbaar** via de serve-logs. Wat ontbreekt is aggregatie. - -### 4.2 Wat de server NIET kan loggen (agent-kant = "waarom") -De keuze "find_impact vs find" wordt in het **LLM-hoofd** van de agent gemaakt, vóór de -tool-aanroep. De server ziet alleen de uitkomst. Om het "waarom" te vangen: - -| Laag | Wat loggen | Hoe | -|------|-----------|-----| -| Server | tool-callfrequentie per type + per taal + outcome (ok/fout) | structured counter/metrics naast tracing; bv. `tool_calls{tool="find_impact",lang="csharp",outcome="ok"}` | -| Server | of `find_impact` faalde door `!is_available` vs `No symbol indexer` | aparte outcome-labels op de counter | -| Agent-harness (opencode/claude) | de tool-selectie-reasoning vóór de call | opencode-session-logs / een wrapper die de assistant-tekst vóór tool_usecapt met "find_impact\|find\|search" | -| Eval-set | 20 vaste queries → welke tool wordt gekozen | herhaalbare harness-run (zie 4.4) | - -### 4.3 Instrumentatie-voorstel (klein, niet-invasief) -1. **Tally in serve-modus:** een in-memory `HashMap<(tool, language, outcome), u64>`, - exposed via `status kind=index` of een nieuw `/metrics`-veld. Laag risico, lokaal in - `CodesearchService`. Bewijst de frequentie-kloof kwantitatief. -2. **Outcome-differentiatie:** onderscheid `Ok` / `NoIndexer` / `HelperUnavailable` / - `Empty` bij `find_impact` — toont aan of het falen (§3.5) de oorzaak is. - -### 4.4 Repro-harness (deterministisch) -Een klein script/set prompts (20 stuks) met mixed intent: -- 8× "who calls / what breaks" (zou → find_impact) -- 6× "find code about X" (zou → search semantic) -- 6× "where is X defined / imports" (zou → find definition/imports) - -Draaien tegen een C# repo **met** scip-csharp én een C# repo **zonder**. Tellen welk % -"who calls" naar find_impact gaat. Vóór fix = baseline, na fix = meting. - ---- - -## 5. (b) Hypotheses (systematisch afgelopen) - -| # | Hypothese | Bewijs nu | Status | -|---|-----------|-----------|--------| -| H1 | Tool-descriptions/afbakening onduidelijk: `find_impact` framt zichzelf als C#-only en raadt `find kind=usages` aan | §3.2 — tool-desc bevat actieve doorverwijzing weg | **Sterk ondersteund** | -| H2 | `find` presenteert zich als de algemene weg; geen caveat dat `find_impact` preciezer is | §3.3 — find-desc "find all call-sites" zonder drempel | **Sterk ondersteund** | -| H3 | Server-instructies routeren "who calls X?" voor niet-C# expliciet weg van find_impact | §3.1 — regels 7931/7945 | **Sterk ondersteund** | -| H4 | Overlappende affordances: zowel find_impact als find kind=usages beantwoorden "who calls X" → agent kiest de generiekere | §3.2+§3.3 combi | Ondersteund (gevolg van H1+H2) | -| H5 | Server-side routing/ranking verbergt find_impact | §4.1 — geen routering die find_impact verbergt; tool is altijd geregistreerd | **Verworpen** | -| H6 | Deploy-realiteit: zonder scip-csharp faalt find_impact → agent leert vermijden | §3.5 — is_available-error | Ondersteund (versterkt H1 voor niet-C#-deploy) | -| H7 | `search(semantic)` als DEFAULT schuift find_impact naar de marge | §3.1 regel 7944 | Ondersteund (zwakker, secundair) | - -**Conclusie H1–H4+H6 zijn allemaal ondersteund en versterken elkaar** → de oorzaak is -multicausaal maar concentreert zich in **framing/afbakening** (beschrijvingen + instructies), -niet in server-routing (H5 verworpen). - ---- - -## 6. (c) Vermoedelijke oorzaak — geïsoleerd - -> **De agent mijdt `find_impact` niet ondanks, maar **door** de documentatie.** - -Eén samengestelde oorzaak, drie dragers: - -1. **Zelf-ontmoedigende tool-description** (`mcp/mod.rs:6236`): `find_impact` zegt letterlijk - "For Rust/Python/Go/etc., use `find` with `kind=usages`". Een agent die deze tekst leest - vóór tool-selectie, volgt die instructie op — correct gedrag, foute uitkomst. -2. **Asymmetrische framing**: `find kind=usages` (4611) claimt zonder voorbehoud "find all - call-sites and references"; `find_impact` geeft zichzelf een taal-drempel. De generiekere - tool wint bij ambiguity. -3. **Server-instructies versterken** (7915-7953): routeert "who calls X?" voor niet-C# - expliciet naar `find kind=usages`, en positioneert `search(semantic)` als default. - -**Dus: het probleem zit in de tekstlaag (descriptions + INSTRUCTIONS_TEMPLATE), niet in -code-logica of routing.** Dat maakt het goed te fixen, maar ook makkelijk te onderschatten -— de "fix" is bewerken van strings, geen refactor. H6 (deploy-falen) is een versterker: -zelfs als de tekst is herzien, blijft `find_impact` falen op een serve-hub zonder scip-csharp; -dat moet via §7-optie B (delegatie) of de losse TS-SCIP-track worden opgelost. - ---- - -## 7. (d) Fix-opties (gefaseerd, niet blind — kies na diagnose-bevestiging) - -### Optie A — Tool-descriptions + instructies herzien (kleinste, eerste stap) -**Wat:** -- `find_impact`-desc (6236): verwijder de actieve doorverwijzing "use find kind=usages". - Hernoem naar taal-neutraal: "Precision symbol impact via SCIP where available; falls back - to lexical matching for languages without a SCIP backend." Maak van SCIP een bonus, niet - een voorwaarde in de framing. -- `find`-desc (4611): voeg bij `usages` een caveat — "lexical/text-based; for IDE-precise - call-graphs use `find_impact`". -- `INSTRUCTIONS_TEMPLATE` (7931/7945): routeer "who calls X?" → `find_impact` als **default**, - niet als C#-uitzondering. `find kind=usages` als fallback alleen als find_impact geen index heeft. -- README (307-321): maak find_impact de aanbevolen call-graph-tool, scip-csharp als - "precision boost" i.p.v. harde vereiste in de framing. - -**Voorspeld effect:** bij de repro-harness (§4.4) stijgt het find_impact-aandeel voor -"who calls X" aanzienlijk — mits een SCIP-index aanwezig is (want anders faalt hij, H6). -**Risico:** op niet-C# repos zonder backend blijft hij falen → agent ziet errors → A alleen -is onvoldoende; combineer met B of de TS-track. - -### Optie B — `find kind=usages` transparant delegatie naar SCIP (middel, structureel) -**Wat:** in `find_usages` (achter `find kind=usages`), detecteer of er een -`SymbolIndexer` voor de betreffende taal/repo beschikbaar + has_index is. Zo ja: roep -`indexer.find_references()` aan (het SCIP-pad) en voeg die resultaten bovenop/ipv de -lexicale match. Zo nee: huidige tekst-based fallback. - -**Effect:** de agent hoeft niets te kiezen — `find kind=usages` wordt automatisch precies -waar SCIP beschikbaar is._lost de asymmetrie (H2) op zonder de agent te belasten. Houdt -`find_impact` als expliciete "geef me alleen SCIP"-tool voor agents die dat willen forceren. -**Risico:** "transparente" upgrade kan verrassingen geven (andere resultaat-volumen/ --volgorde); documenteer + feature-flag (`CODESEARCH_FIND_DELEGATES_TO_SCIP`, default aan). -Complexiteit: ~1 functie in `find_usages` + taal-detect per query (de file-ext logica uit -`find_impact` 6295 hergebruiken, maar dan generiek). - -### Optie C — Tools samenvoegen (grootst, breekend) -**Wat:** één `find_references`-tool (of `find_impact` hernoemen) die altijd SCIP-voorrang -geeft en valt terug op lexicaal. `find kind=usages` afschaffen of als alias behouden. -**Effect:** elimineert de ambiguity volledig (H4 weg). Maar: breaking voor agents/harnesses -die `find kind=usages` aanroepen; migratiekosten; grotere review. -**Risico:** backward-compat, alias-beheer. **Alleen kiezen als A+B onvoldoende blijken.** - -### Optie D — Language-aware routing binnen `find` (klein, complementair) -**Wat:** de `suggested_tool`-note die de server nu emit (§3.4 meta) uitbreiden: bij een -"who calls"-aardige query op een C# repo, suggesteer `find_impact` i.p.v. `find kind=usages`. -**Effect:** nudges de agent in-session, zonder tool-schema's te raken. -**Risico:** klein; louter aanvullend op A/B. - -### Aanbevolen volgorde -1. **A eerst** (tekst-laag, goedkoop, direct meetbaar in repro-harness). -2. **B als structurele oplossing** (lost H6 op: ook zonder find_impact-aanroep krijgt de - agent SCIP-kwaliteit via de vertrouwde `find`-tool). -3. C alleen als A+B in de eval niet voldoen. -4. D als finishing touch. -De losse TS-SCIP-track (`PLAN_TYPESCRIPT_SCIP.md`) breidt de **dekking** van find_impact uit -(meer talen met écht SCIP); dit diagnose-plan los de **keuze**-bias op. Beide zijn -complementair. - ---- - -## 8. Review-sectie — open ontwerpkeuzes & risico's - -| Keuze | Opties | Risico / afweging | -|-------|--------|-------------------| -| Verwijderen vs. verzachten van "use find kind=usages" in find_impact-desc | hard verwijderen kan agent in niet-C# zonder backend op een falende tool zetten | combineer altijd met B (delegatie) of een duidelijke runtime-foutmelding die wéér naar find_impact... → nee: naar `find kind=usages` als echte fallback (geen cirkel) | -| Delegatie default aan/uit | default AAN = transparante upgrade; default UIT = backward-compat | feature-flag, default aan na evaluatieperiode | -| Meten vóór/na | repro-harness is handmatig vandaag | overweeg een klein geautomatiseerd eval-script in `tests/` of `eval/` | -| `search(semantic)` als DEFAULT-handhaving | verwijderen verzwakt de grep-guard die search beschermt | behouden, maar herformuleer zodat find_impact niet onder "code lookup" valt maar onder "impact/call-graph" als eigen categorie | -| find_impact op repo zonder index | vandaag: error → agent vermijdt | bij delegatie (B) wordt dit onzichtbaar goed; zonder B: betere foutmelding die de agent niet de hele tool laat vermijden | -| Backward-compat van tool-schema's | samenvoegen (C) breekt callers | alleen bij voldoende wins; anders A+B behouden beide tools | - -**Belangrijkste review-waarschuwing:** niet de server-logica is kapot (H5 verworpen) — -de agent volgt de instructies correct. Een "fix" die alleen code-logica aanraakt zonder de -tekstlaag (descriptions/instructies) raakt het hoofdbewijs niet. - ---- - -## 9. Acceptatiecriteria voor de diagnose (waneer is "oorzaak bewezen"?) -- [ ] Repro-harness (§4.4) levert een baseline: % "who calls X" → find_impact vóór fix. -- [ ] Server-tally (§4.3) toont kwantitatief de kloof (find_impact vs find kind=usages). -- [ ] H1–H4+H6 bevestigd, H5 verworpen — met code-citaten uit §3. -- [ ] Eén gekozen fix-optie (A en/of B) geïmplementeerd → repro-harness na fix toont - meetbare stijging van find_impact-aandeel (bij A) of SCIP-kwaliteit bij find (bij B). -- [ ] Geen regressie: bestaande `find kind=usages` op niet-C# repo's blijft werken. - ---- - -## 10. Verwijzingen -- Server-instructies (agent system-prompt bron): `src/mcp/mod.rs:7915-7953` (`INSTRUCTIONS_TEMPLATE`) -- `find_impact`-description + handler: `src/mcp/mod.rs:6236-6380` (taal-detect 6291-6329, is_available 6332-6347) -- `find`-description + dispatch: `src/mcp/mod.rs:4611-4670` -- `suggested_tool`-meta (search-result nudge): emit in zoekresultaat-output -- Publieke docs: `README.md:307-321` (find_impact C#-only framing), `README.md:241-329` (tool reference) -- Instructie-test guard: `src/mcp/mod.rs:407-433` (`test_no_deprecated_tool_aliases_in_instructions`) -- Complementair plan: `PLAN_TYPESCRIPT_SCIP.md` (dekking-uitbreiding, niet keuze-bias) diff --git a/PLAN_TYPESCRIPT_SCIP.md b/PLAN_TYPESCRIPT_SCIP.md deleted file mode 100644 index 34fd9061..00000000 --- a/PLAN_TYPESCRIPT_SCIP.md +++ /dev/null @@ -1,239 +0,0 @@ -# PLAN — TypeScript SCIP-indexering (find_impact + call-graph voor TS) - -> **Status:** PLANNING — geen code geschreven. Dit document is het oppakpunt voor de implementatie. -> **Doel:** `find_impact` en de call-graph voor TypeScript (.ts/.tsx) laten werken zoals nu voor C#, -> door de bestaande C#-SCIP-pijplijn te spiegelen met Sourcegraph `scip-typescript`. -> **Branch-target:** PRs tegen `develop` (zie AGENTS.md gitflow). - ---- - -## 1. Doel & scope - -**In scope** -- `TypeScriptSymbolIndexer` implementeert het bestaande `SymbolIndexer`-trait, gevoed door `scip-typescript` (Sourcegraph, npm CLI). -- `find_impact` (MCP-tool) routeert `.ts`/`.tsx`/`.mts`/`.cts` bestanden naar de TS-indexer. -- Single-pass indexering: `rebuild()` schrijft defs **en** refs in één run naar LMDB (geen two-phase lazy model nodig — zie §3). -- File-watcher pakt `.ts`/`.tsx`-wijzigingen op en triggert een TS-debounced rebuild. -- Tests bewijzen dat `find_impact` op een TS-symbool alle call-sites teruggeeft. - -**Out of scope (follow-up)** -- TS in de release-bundel shippen (`-with-ts` archive / `helpers/typescript/`) — optioneel, scip-typescript is een npm-package dus `npx` volstaat op de host. -- Incrementele `RebuildScope::Files` voor TS (single-pass maakt full-rebuild op kleine/ middelgrote repo's al snel genoeg; incrementeel is een latere optimalisatie). - ---- - -## 2. Hoe C#-SCIP nu werkt (baseline voor spiegeling) - -De TS-feature moet dezelfde raakvlakken gebruiken. Dit is de C#-status quo: - -### 2.1 Trait + registry (`src/symbols/mod.rs`) -- **Trait `SymbolIndexer`** (regels 113-168): `language()`, `rebuild(repo_path, db_path, RebuildScope)`, `find_references(db_path, symbol)`, `find_references_by_position(db_path, file, line)`, `index_age()`, `is_available()`, `has_index()`, `applies_to(repo_path)`, `as_any()`. -- **`SymbolIndexerRegistry`** (regels 172-232): houdt `Vec>`. `new()` (regel 181) registreert **uitsluitend** `CSharpSymbolIndexer::new()`. `get(language)` is case-insensitive. Methodes: `available_languages()`, `installed_languages()` (filter op `is_available()`), `has_index_for()`, `indexed_languages()`. -- **Gedeelde types:** `SymbolReference{file,start_line,end_line,kind}`, `FindImpactResult`, `SymbolIndexError`, `RebuildScope` (Full | Project(PathBuf) | Files{changed,deleted}), `RebuildSummary`, `PrewarmSummary`. - -### 2.2 C#-adapter (`src/symbols/csharp.rs`, 1740 regels) -`struct CSharpSymbolIndexer` implementeert het trait: -- `detect_helper()` / `resolve_helper_path()` / `validate_helper_path()` (239-353) — zoekt `scip-csharp` via env `CODESEARCH_SCIP_CSHARP` of `helpers/csharp/`. -- `find_solution(repo)` / `find_csproj_for_file(repo, file)` (355-391) — entrypoint-detectie (.sln/.csproj). -- `open_scip_env(db_path)` (398-429) — opent LMDB env in `db_path/scip/`, pre-createert 5 named DBs. -- `invoke_index_helper(...)` (433-504) — spawnt `scip-csharp index --solution X --output Y [--filter-project Z]`. -- `invoke_find_refs_helper()` (509-609), `invoke_batch_find_refs_helper()` (954-1033) — **lazy ref-resolutie** subcommands. -- Trait-impl (1124-1641): `language()`="csharp", `applies_to()` checkt .sln/.csproj, `is_available()` checkt `detect_helper()`. - -### 2.3 Two-phase lazy reference model (C#-specifiek — TS doet dit ANDERS) -1. `rebuild()` → `scip-csharp index` emit **alleen definities** → snel. -2. `find_references()` resolvet refs on-demand: defs uit LMDB → cache-check → cache-miss → `scip-csharp find-refs` voor dat symbool → cache resultaat. -3. Pre-warm: `scip-csharp batch-find-refs` resolvet alle refs in één workspace-sessie. -- **C# helper output = custom JSON** (NIET standaard SCIP protobuf), geparseerd door `parse_json_index` in `scip_parse.rs` (regels 136-197). - -### 2.4 LMDB-schema (`db_path/scip/`, 5 named DBs — keys namespaced door SCIP-symbol-scheme taal-prefix) -| DB | key | value | -|----|-----|-------| -| `scip_symbols` | full SCIP symbol | bincode `Vec` | -| `scip_meta` | `"last_rebuild_ts"` | timestamp — **let op: NIET per-taal!** | -| `scip_positions` | `"file:line"` | `Vec` | -| `scip_simple_names` | simple name | `Vec` | -| `scip_ref_cache` | symbol | bincode `Vec` | - -### 2.5 Dispatch-punten die vandaag HARDCODED op C# staan (moeten generaliseren of een TS-tak krijgen) -| Locatie | Regel | Wat het doet | Voor TS | -|---------|-------|--------------|---------| -| `src/mcp/mod.rs` find_impact | 6296 | file-ext → language: alleen `"cs"` | voeg `"ts"/"tsx"/"mts"/"cts"` → LANG_TYPESCRIPT | -| `src/index/manager.rs` tracking | 1124, 1138, 1152 | trackt `.cs` modified/deleted/rename | voeg `.ts`/`.tsx` tracking toe | -| `src/index/manager.rs` debounce-flush | 1236 | `reg.get(LANG_CSHARP)` (hardcoded) | dispatch generiek over registry OF parallelle TS-tak | -| `src/index/manager.rs` notifier-type | — | `CSharpRebuildNotifier` callback | generaliseer of `TsRebuildNotifier` | -| `src/serve/mod.rs` Phase-3 pre-warm | 1045 | `symbol_registry.get(LANG_CSHARP)` | pre-warm loop over alle registry-talen | -| `src/serve/mod.rs` status | — | `CSharpIndexStatus::None/Ready` | generaliseer naar per-taal status-map | - ---- - -## 3. Het TS-pad: spiegelen met scip-typescript - -### 3.1 Kritieke verschillen met C# -| Aspect | C# (scip-csharp) | TS (scip-typescript) | -|--------|------------------|----------------------| -| **Output-formaat** | custom JSON | **standaard SCIP protobuf `.scip`** | -| **Referentiemodel** | two-phase lazy (defs dan refs) | **single-pass** (defs + refs samen) | -| **Entrypoint** | `.sln` / `.csproj` | `tsconfig.json` | -| **Runtime** | self-contained .NET exe | Node CLI: `npx scip-typescript index` | -| **find_references** | on-demand subprocess + cache | **alleen LMDB-lees** (geen subprocess) | - -### 3.2 Consequentie voor de implementatie -1. **Protobuf-parse nodig.** scip-typescript is fixed binary-formaat → optie B (eigen TS-helper die JSON emit) is niet haalbaar. Keuze: de `scip` Rust-crate (Sourcegraph) toevoegen + een parser in nieuw `src/symbols/scip_proto.rs` die `.scip` → zelfde `ScipIndex`-shape mapt als `scip_parse.rs` nu voor JSON doet. Daarna is alle storage/resolution-code herbruikbaar. -2. **Geen two-phase.** TS `rebuild()` vult in één pass `scip_symbols` + `scip_positions` + `scip_simple_names` én de refs. `find_references()` leest alleen LMDB (snel, geen subprocess). `scip_ref_cache` is voor TS leeg/overbodig — schrijven kan geen kwaad (keys namespaced). -3. **`is_available()`** voor TS = detecteer of `scip-typescript` oplosbaar is via env `CODESEARCH_SCIP_TYPESCRIPT` (pad naar binary) of via `npx` op PATH + Node aanwezig. -4. **`applies_to()`** voor TS = zoek een `tsconfig.json` in `repo_path` (root of één niveau diep). - ---- - -## 4. Betrokken files & functies (concreet) - -### 4.1 Nieuwe files -| File | Inhoud | -|------|--------| -| `src/symbols/scip_proto.rs` | `parse_scip_protobuf(bytes) -> ScipIndex` via `scip` crate. Herbruikt `ScipReference`/`ScipIndex` uit `scip_parse.rs`. | -| `src/symbols/typescript.rs` | `struct TypeScriptSymbolIndexer` impl `SymbolIndexer`. Mirrot van `csharp.rs` structuur: `detect_helper()`, `find_tsconfig(repo)`, `open_scip_env()` (hergebruik), `invoke_index_helper()`, trait-impl. | -| `tests/symbols_typescript_test.rs` | Gated integratie-test (zelfde gate-patroon als `symbols_csharp_test.rs`), TS-fixture. | -| `tests/fixtures/ts-sample/` | Klein TS-project: `tsconfig.json` + 2-3 `.ts` files met een functie + call-sites. | - -### 4.2 Te wijzigen files (exacte raakvlakken) -| File | Wijziging | -|------|-----------| -| `Cargo.toml` | voeg `scip` dependency toe (Sourcegraph crate) | -| `src/symbols/mod.rs` regel 181 | `SymbolIndexerRegistry::new()` registreer óók `typescript::TypeScriptSymbolIndexer::new()` | -| `src/symbols/mod.rs` | voeg `pub mod typescript;` + `pub mod scip_proto;` toe | -| `src/constants.rs` | `LANG_TYPESCRIPT="typescript"`, `SCIP_TYPESCRIPT_HELPER_ENV="CODESEARCH_SCIP_TYPESCRIPT"`, `SCIP_TYPESCRIPT_HELPER_NAME="scip-typescript"`, `TS_DEBOUNCE_MS` | -| `src/mcp/mod.rs` regel 6296 | find_impact auto-detect: map `"ts"/"tsx"/"mts"/"cts"` → `LANG_TYPESCRIPT` | -| `src/mcp/mod.rs` regel 6236 | update tool-description (nu: "C# today") → voeg TS toe | -| `src/index/manager.rs` regels 1124/1138/1152 | voeg `.ts`/`.tsx`-tracking velden toe (`ts_files_modified/deleted/last_event_time`) | -| `src/index/manager.rs` regel 1236 | dispatch: óf registry-loop, óf parallelle TS-tak na C#-tak | -| `src/index/manager.rs` notifier | generaliseer `CSharpRebuildNotifier` naar generieke `SymbolRebuildNotifier` (boxed callback) | -| `src/serve/mod.rs` regel 1045 | Phase-3 pre-warm: itereren over registry in plaats van hardcoded `LANG_CSHARP` | -| `src/serve/mod.rs` | vervang `CSharpIndexStatus` door `HashMap` (per-taal) | - -### 4.3 Niet-wijzigen (herbruikbaar) -- `src/symbols/scip_parse.rs` structs (`ScipReference`, `ScipIndex`) — de protobuf-parser mapped hiernaartoe. -- LMDB-schema (de 5 named DBs) — keys zijn namespaced door SCIP-symbol-scheme, dus C# en TS co-existeren in dezelfde `db_path/scip/`. -- `RebuildScope`, `RebuildSummary`, `PrewarmSummary`, `SymbolReference`, `FindImpactResult` types. - ---- - -## 5. Per-taal indexer-selectie (hoe taal-bepaling werkt) - -Twee routes die beide TS moeten ondersteunen: - -### 5.1 Expliciet (MCP find_impact `request.language`) -`SymbolIndexerRegistry::get(language)` is case-insensitive en retourneert de indexer waarvan `language()` overeenkomt. `LANG_TYPESCRIPT="typescript"` → `registry.get("typescript")` werkt automatisch zodra geregistreerd. - -### 5.2 Auto-detect (file-extensie) -`src/mcp/mod.rs:6296` — huidige map is **enkel** `"cs" → LANG_CSHARP`, else fallback naar eerste `installed_languages()`. **Toevoegen:** -```rust -match ext { "cs" => LANG_CSHARP, "ts"|"tsx"|"mts"|"cts" => LANG_TYPESCRIPT, _ => /* fallback */ } -``` -Fallback = huidig gedrag (eerste installed language) — ongewijzigd. - -### 5.3 Applicability (welke indexer pakt een repo op?) -- `applies_to(repo_path)` per indexer: C# checkt `.sln`/`.csproj`, TS checkt `tsconfig.json`. -- `installed_languages()` filtert op `is_available()` (helper gevonden). Een host zonder Node/scip-typescript ziet TS simpelweg niet — geen crash. - ---- - -## 6. Implementatie-stages (volgorde voor PR(s)) - -| # | Stage | Doel | Validering | -|---|-------|------|------------| -| 1 | Protobuf-binding | `scip` crate + `scip_proto.rs::parse_scip_protobuf()` | unit-test: fixture `.scip` file → `ScipIndex` met verwacht # defs/refs | -| 2 | TypeScriptSymbolIndexer | nieuw `typescript.rs`, implementeert trait | `cargo check` + `cargo clippy -D warnings` | -| 3 | Registratie + constants | `mod.rs:181` registreer TS; `constants.rs` lang/env | `installed_languages()` bevat "typescript" als Node aanwezig | -| 4 | find_impact auto-detect | `mcp/mod.rs:6296` map TS-extensies | handmatige smoke: find_impact op een TS-file | -| 5 | File-watcher TS-tracking | `manager.rs` `.ts`/`.tsx` + dispatch | bewerk een `.ts` → debounce-flush triggert rebuild | -| 6 | Tests | `tests/symbols_typescript_test.rs` + fixture | `cargo test --test symbols_typescript_test` groen | -| 7 | Pre-warm + status generaliseren | `serve/mod.rs` registry-loop | startup log toont TS pre-warm | -| 8 | (optioneel) Release-bundling | `release.yml` `-with-ts` | archive bevat scip-typescript binary | - -Stages 1-6 zijn de MVP (find_impact werkt op TS). 7-8 zijn afronding. - ---- - -## 7. Test-strategie: bewijs dat find_impact op een TS-symbool alle call-sites teruggeeft - -### 7.1 Fixture-ontwerp (`tests/fixtures/ts-sample/`) -``` -ts-sample/ - tsconfig.json # compilerOptions, minimal - src/ - math.ts # export function add(a, b) ← TARGET definitie - consumer.ts # import { add }; add(1,2); add(3,4) ← 2 call-sites - other.ts # import { add }; const r = add(5,6) ← 1 call-site -``` -Doel: `add` heeft 1 definitie + 3 call-sites verdeeld over 2 files. - -### 7.2 Integratie-test (`tests/symbols_typescript_test.rs`) -Gated (zelfde patroon als `symbols_csharp_test.rs`: skip als `scip-typescript`/Node niet oplosbaar, geen real embedding nodig). Test-flow: -1. `TypeScriptSymbolIndexer::new()` -2. `.rebuild(&fixture_root, db_path, RebuildScope::Full)` → `assert!(summary.ok)` -3. `.has_index(db_path)` → `true` -4. `.find_references(db_path, "add")` (via simple_name) → `assert_eq!(refs.len(), 4)` (1 def + 3 calls) OF via full SCIP-symbol key -5. `.find_references_by_position(db_path, "src/math.ts", )` → retourneert de `add`-symbol key -6. Cross-check: voor elke call-site file komt deze voor in `refs.iter().map(|r| r.file)` - -### 7.3 find_impact end-to-end (optioneel, handmatig) -Na opstarten van `codesearch serve` met de TS-fixture als project: roep de `find_impact` MCP-tool aan met `{file:"src/math.ts", line:}` en verifieer dat het resultaat overeenkomt met de integratie-test (4 occurrences over 3 files). - -### 7.4 Negative tests -- `find_references` op een onbekend symbool → lege `Vec`, geen panic. -- `.is_available()` op een host zonder Node → `false`; `installed_languages()` bevat geen "typescript". - ---- - -## 8. Review — openstaande ontwerpkeuzes (beslissen vóór/ten tijde van implementatie) - -### 8.1 `scip_meta` is NIET per-taal (design-issue) -`scip_meta` gebruikt key `"last_rebuild_ts"` zonder taal-prefix. Bij twee talen in dezelfde `db_path/scip/` overschrijven C# en TS elkaars timestamp. **Optie:** key namespacen `"last_rebuild_ts:csharp"` / `"last_rebuild_ts:typescript"`. Niet-breaking voor lezers die via `index_age()` gaan. **Beslissing:** namespacen — lokaal in `typescript.rs` een eigen key gebruiken, en later C# migreren. - -### 8.2 File-watcher dispatch: generaliseren vs. parallelle tak -- **Optie A (generiek):** vervang hardcoded `reg.get(LANG_CSHARP)` (manager.rs:1236) door een loop `for lang in registry.indexed_languages()`. Schoon, schaalbaar naar meer talen, maar raakt `CSharpRebuildNotifier`-type (moet generiek `SymbolRebuildNotifier` worden) — grotere refactor. -- **Optie B (parallelle tak):** voeg een tweede `if`-blok voor TS toe, spiegelend het C#-blok. Minder netjes, lokaal, lager risico. -- **Beslissing:** start met Optie B (snel MVP), refactor naar A zodra er een derde taal komt. Documenteer als TODO. - -### 8.3 scip-typescript distributie: `npx` vs. gebundelde binary -- C# shipt een self-contained exe in `helpers/csharp/` + `-with-csharp` release-archives. -- scip-typescript is een npm-package: `npx scip-typescript` werkt als Node op PATH staat. Geen bundling nodig voor development. Voor offline/air-gapped deploy: optie om `npm pack`-tarball te bundelen (follow-up, niet MVP). -- **Beslissing:** MVP = `npx` (env `CODESEARCH_SCIP_TYPESCRIPT` voor override-pad). Bundling = out of scope (§1). - -### 8.4 Incrementele rebuild (RebuildScope::Files) voor TS -C# ondersteunt `Files{changed,deleted}` via csproj-groepering + `--filter-project`. scip-typescript heeft geen file-filter flag — herbouwt steeds de hele tsconfig-projectroot. **Beslissing:** MVP ondersteunt alleen `Full`; `Files` valt terug op `Full` (log + proceed). Voor grote monorepo's is dit later te optimaliseren (per-tsconfig groeperen, zie §8.5). - -### 8.5 Monorepo met meerdere tsconfig.json -`applies_to()` zoekt nu één `tsconfig.json`. Een monorepo met `packages/*/tsconfig.json` vereist het C#-equivalent van `find_csproj_for_file` → een `find_tsconfig_for_file(repo, file)`. **Beslissing:** MVP pakt root-tsconfig; per-file-tsconfig-resolutie = follow-up. In scope zetten als de test-fixture dat meteen nodig maakt. - -### 8.6 Pre-warm: heeft TS het nodig? -TS heeft geen two-phase lazy model → `rebuild()` populate direct alle refs → geen `batch-find-refs` pre-warm nodig. De registry-loop in `serve/mod.rs:1045` mag TS dus overslaan of gewoon `rebuild` aanroepen als index ontbreekt/stale is. **Beslissing:** registry-loop roept per indexer een `prewarm()`-methode aan; C# doet zijn batch-find-refs, TS is no-op (of `rebuild` als index koud). Voeg optionele default-methode `prewarm()` toe aan het trait. - -### 8.7 `scip` crate keuze -Sourcegraph publiceert een `scip` Rust-crate (protobuf bindings + helpers). Alternatief: handmatige `prost`-build tegen de `.proto`. **Beslissing:** gebruik de `scip` crate (onderhouden, zelfde schema als scip-typescript output). Lock versie in `Cargo.toml`; als de crate afwijkt, val terug op `prost`-build. - ---- - -## 9. Acceptatie-criteria (MVP = stages 1-6) -- [ ] `cargo check` + `cargo clippy -D warnings` groen. -- [ ] `cargo test --test symbols_typescript_test` groen op een host met Node + scip-typescript. -- [ ] `find_impact` MCP-tool retourneert voor een TS-functie alle call-sites (≥3 over 2 files in de fixture). -- [ ] `find_impact` auto-detect routeert `.ts`/`.tsx` naar TS-indexer (geen C#-fallback). -- [ ] `installed_languages()` bevat "typescript" als Node aanwezig, niet anders. -- [ ] Host zonder Node: geen crash, TS-indexer gewoon afwezig. -- [ ] C#-pijplijn ongewijzigd werken (geen regressie — bestaande C#-tests groen). - ---- - -## 10. Verwijzingen -- C# trait + registry: `src/symbols/mod.rs:113-232` -- C# adapter (referentie-impl): `src/symbols/csharp.rs` -- JSON-parser (shape om naartoe te mappen): `src/symbols/scip_parse.rs:136-197` -- find_impact MCP-tool: `src/mcp/mod.rs:6238-6369` (taal-detect 6291-6329) -- File-watcher dispatch: `src/index/manager.rs:1124-1340` -- Startup pre-warm: `src/serve/mod.rs:1045` -- Constants: `src/constants.rs` (LANG_CSHARP, SCIP_CSHARP_*, HELPERS_SUBDIR) -- Bestaande tests: `tests/symbols_csharp_test.rs`, `helpers/csharp/tests/IndexerTests.cs` -- scip-typescript (Sourcegraph): https://github.com/sourcegraph/scip-typescript -- scip Rust-crate: https://github.com/sourcegraph/scip-rust diff --git a/README.md b/README.md index 8609c987..a09b30db 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,21 @@ spaces are model-specific. Keep the same model selected for later indexing runs. Search rejects a `--model` value that differs from the indexed model and points to the required `--force` rebuild instead of mixing incompatible vector spaces. +In **serve** mode the model is resolved **per repository**, from each index's own +metadata, not from a hub-wide setting: a hub may hold indexes built with +different models, and every query is embedded with the model of the repo it +targets (mixed-model groups are fine). `codesearch serve --model ` sets a +**default for newly created indexes**: a repo added without an explicit model +(e.g. `codesearch index add` with no `--model`, delegated to serve) is indexed +with it, and it is reported in `GET /status` as `default_model`. It never +overrides an index that already records its own model — to change an existing +repo's model, re-index that repo +(`codesearch --model index --force`) and restart serve. A repo +whose `metadata.json` records no model (a legacy index built before the +recording contract) is queried with the built-in 384-dim default rather than the +serve default, and the search response carries a warning naming the assumed model +and the re-index command. + ## MCP Configuration codesearch connects to AI agents via MCP. Two modes: @@ -222,10 +237,11 @@ OpenCode: put this in the user-level `~/.config/opencode/AGENTS.md` (applies acr **Claude Code specifically** tends to ignore this advice more than other clients — its MCP tool schemas are deferred (an extra `ToolSearch` call is needed before codesearch tools are even callable), while Grep/Glob are always fully loaded and zero-friction, and spawned subagents don't inherit `AGENTS.md` or the MCP `initialize` instructions at all. -To make the preference **structural** instead of advisory, this repo ships three Claude Code `PreToolUse` hooks: +To make the preference **structural** instead of advisory, this repo ships five Claude Code hooks (four `PreToolUse` guards plus one `PostToolUse` companion): -- **`grep-guard`** — on `Grep`. Blocks a grep against an in-repo path when codesearch covers that repo (a local `.codesearch.db` at the git root, or a `CODESEARCH_SERVER` env var for remote-serve setups), with a message telling the model how to load and call codesearch instead. Grep is auto-allowed **only when the serve hub is genuinely down**, established by a live probe of the unauthenticated `/healthz` endpoint (`CODESEARCH_SERVER` > `127.0.0.1:$CODESEARCH_SERVE_PORT` > `127.0.0.1:39725`); only a connection-level failure counts as down. A low-confidence or empty codesearch result is a *successful* call meaning "reformulate the query", so it does **not** open the escape hatch — the deny message steers to `find`/`explore`/a single clean term instead. Greps outside the current repo are never blocked, and the hook fails open (never traps the model). -- **`subagent-preamble`** — on `Agent` (the subagent-spawn tool). Prepends a short codesearch preamble to every subagent prompt, since subagents otherwise don't inherit `AGENTS.md` or MCP instructions at all. +- **`grep-guard`** — on `Grep`. Blocks a grep against an in-repo path when codesearch covers that repo (the target repo — resolved from the grep target's own git root — is registered with the serve hub in `~/.codesearch/repos.json`, or a `CODESEARCH_SERVER` env var is set for remote-serve setups), with a message telling the model how to load and call codesearch instead. Grep is auto-allowed **only when the serve hub is genuinely down**, established by a live probe of the unauthenticated `/healthz` endpoint (`CODESEARCH_SERVER` > `127.0.0.1:$CODESEARCH_SERVE_PORT` > `127.0.0.1:39725`); only a connection-level failure counts as down. A low-confidence or empty codesearch result is a *successful* call meaning "reformulate the query", so it does **not** open the escape hatch — the deny message steers to `find`/`explore`/a single clean term instead. Greps outside any registered repo are never blocked, and the hook fails open (never traps the model). +- **`edit-guard`** — on `Edit`/`Write`/`MultiEdit`. Blocks an edit to a file in a codesearch-registered repo until codesearch was consulted for that exact path within the last 5 minutes — `find_impact` for SCIP-backed languages (`.cs .ts .tsx .mts .cts`), `find(kind="usages")` for everything else — making the caller-aware-editing protocol structural. Its companion **`edit-guard-post`** (`PostToolUse`) records the markers on every `find_impact` call and every `find(kind="usages")`; any outcome counts ("no results" included), so the guard can never wedge. Unregistered repos, non-git paths and hook failures fail open. +- **`subagent-preamble`** — on `Agent` (the subagent-spawn tool). Prepends a short codesearch preamble to every subagent prompt, since subagents otherwise don't inherit `AGENTS.md` or MCP instructions at all — including the edit-guard protocol above. - **`web-guard`** — on `WebSearch`/`WebFetch`. When you have remote documentation projects mounted (`codesearch remote mount`, e.g. `cloud/inriver`, `cloud/example-dam`), it blocks the first web call with guidance to search those indexed mounts first — often more precise and current than the open web. Same 5-minute retry-escape; when no mounts are configured it does nothing. Install (idempotent — user scope applies to every project; `--project` is this repo only): @@ -237,7 +253,7 @@ codesearch hooks claude install --project # project scope (./.claude) The native command embeds the hook scripts in the binary (no source tree needed) and merges the registrations into `settings.json`. The equivalent from-source installers still live in [`integrations/claude-code/`](integrations/claude-code/) (`install.ps1` / `install.sh`) if you'd rather run them directly. -Note: the grep-guard detects "codesearch is available **for this repo**" via a local `.codesearch.db` or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory). For a remote-serve setup with no local index, set `CODESEARCH_SERVER` to opt back into enforcement. +Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` override) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. ## MCP Tools Reference @@ -320,6 +336,8 @@ Returns a list of references with `file`, `start_line`, `end_line`, and `kind` ( > **Note:** SCIP precision requires the `-with-csharp` release variant (or a separately installed `scip-csharp` helper) for C#, and `npx` (with `scip-typescript`, fetched on first use) on the host's PATH for TypeScript. Without a backend for a language, `find_impact` returns a clear message — use `find kind="usages"` as the lexical fallback. See [C# Semantic Search](#c-semantic-search). +On a running `codesearch serve` the same lookup is available over plain REST — `POST /find-impact` with the identical request body, no MCP session required (same auth class as the other read-only REST endpoints). + ### `status` — Index Info | Parameter | Type | Description | @@ -437,7 +455,7 @@ The install target is resolved with `git rev-parse --git-path hooks`, so it hono ### Claude Code Guard Hooks -`codesearch hooks claude install` (`--project` for repo scope) installs the `PreToolUse` guard hooks that steer agents to codesearch before `Grep`/`WebSearch`/`WebFetch`. See [Agent Guidance](#agent-guidance-making-agents-use-codesearch-not-grep) above for what each guard does. +`codesearch hooks claude install` (`--project` for repo scope) installs the four `PreToolUse` guard hooks that steer agents to codesearch before `Grep`/`WebSearch`/`WebFetch`/`Edit`/`Write`/`MultiEdit`, plus the `PostToolUse` marker hook (`edit-guard-post`) that records the consultations `edit-guard` requires. See [Agent Guidance](#agent-guidance-making-agents-use-codesearch-not-grep) above for what each guard does. ### MCP Connection Modes diff --git a/RELEASING.md b/RELEASING.md index 886dfed9..e9108db4 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -49,6 +49,22 @@ git push -u origin release/v1.0.X Create PR → `master`. **Squash merge.** +> **Keep contributors credited** — a squash commit is authored by whoever clicks +> merge, so individual authorship never reaches master, and the GitHub +> contributors list counts only the default branch. Carry the author trailers +> in the squash commit body (review the list first — it may contain device/test +> identities you don't want credited): +> +> ```bash +> scripts/release-coauthors.sh # prints Co-authored-by: lines +> gh pr merge --squash --admin \ +> --subject "release: v1.0.X" \ +> --body "$(scripts/release-coauthors.sh)" +> ``` +> +> Squash-merging via the GitHub web UI instead? Its default squash message +> already appends co-author trailers automatically — no script needed. + ### 3. Tag release ```bash diff --git a/TEST-SCENARIO-remote-mount-semantic-search.md b/TEST-SCENARIO-remote-mount-semantic-search.md deleted file mode 100644 index cfe8b715..00000000 --- a/TEST-SCENARIO-remote-mount-semantic-search.md +++ /dev/null @@ -1,252 +0,0 @@ -# Test Scenario — Semantic findability across remote-mounted doc projects - -Acceptance test for the `features/remote-mount-selection` work: does codesearch -**find the right content in the right mounted doc project — and *not* surface it -where it doesn't belong** — and does the federated `get_chunk` round-trip work -end-to-end (the Stage A `ambiguous_chunk_id` fix). - -The corpus is six product-documentation indexes mounted from the `cloud` peer, a -natural mix of **PIM** and **DAM** products. That overlap (two PIMs, three DAMs) -is exactly what makes findability testable: a PIM concept *should* surface in a -PIM index and *should not* have a genuine match in a DAM index, and vice-versa. - ---- - -## 0. Preconditions - -| # | Check | How | -|---|-------|-----| -| P1 | `serve` is active and the six mounts are present | `status(kind="projects")` → `remote_projects[]` lists `cloud/akeneo`, `cloud/example-dam`, `cloud/bynder`, `cloud/custom-kb`, `cloud/digizuite`, `cloud/inriver` | -| P2 | The `docs` group federates the peer | `status(kind="projects")` → `groups.docs == ["@cloud"]` | -| P3 | **The serve process runs the Stage-A binary** | A remote search result's `chunk_ref` is namespaced `"cloud/inriver:"`, **not** the legacy `"cloud:"`. See ⚠️ below. | - -> ⚠️ **Known state at time of writing:** the live serve still runs a **pre-Stage-A -> binary** — remote results come back with legacy `chunk_ref` `"cloud:"` (no -> alias). Section **D** is therefore the gating regression test: it is expected to -> reproduce the old `ambiguous_chunk_id` bug on the current binary and to pass only -> after serve is rebuilt/restarted with the fixed binary. Redeploy, then re-run. - -### The mounted products (concept map) - -| Mount | Product | Domain | Signature concepts (owns) | -|-------|---------|--------|---------------------------| -| `cloud/inriver` | inriver | **PIM** | entity/link model, variants, channels, syndication, Enrich, Control Center | -| `cloud/akeneo` | Akeneo | **PIM** | product families, attribute groups, categories, reference entities, connectors | -| `cloud/example-dam` | example-dam | **DAM + MO** | Marketing Operations, workflow designer, DAM records, classifications, review/approval | -| `cloud/bynder` | Bynder | **DAM** | asset portal, brand guidelines, Studio, collections, asset workflow | -| `cloud/digizuite` | Digizuite | **DAM** | DAM Center, media renditions, transformations, publishing destinations | -| `cloud/custom-kb` | custom KB | mixed | wildcard — no assumption | - ---- - -## ⚖️ Scoring caveat — READ THIS BEFORE JUDGING RESULTS - -Result `score` is **RRF (Reciprocal Rank Fusion)** — a *rank-based* number, not an -absolute similarity. In calibration the **top hit scored ~0.0476 in every index**, -including one where the query had no genuine match. So: - -- **Never judge findability by the score number.** The top score is ~0.0476 whether - the match is perfect or garbage. -- **Judge findability by the returned content**: does the top-ranked chunk's - `path` + body actually address the queried concept? -- A **true positive** = the top 1–3 chunks are *on-topic* docs for the concept. -- A **true negative** = the top chunks are *off-topic* (release notes, unrelated - features) — the concept simply isn't documented in that product. - ---- - -## A. True positives — semantic recall in the owning product - -Each query is phrased in **different words** than the docs use, so a plain keyword -match would miss it. Semantic search must still surface the right doc. -Run with `search(mode="semantic", project="", query="…", limit=5)`. - -| # | Project | Query (natural language) | PASS = top 1–3 chunks are about… | Calibrated? | -|---|---------|--------------------------|----------------------------------|-------------| -| A1 | `cloud/inriver` | "how are product entities linked to variants and sales channels" | inriver **entity / elastic data model** (e.g. `…/What-is-an-entity.md`, channel/link docs) | ✅ verified — hit `getting-started/elastic-data-model-common-terminology/…What-is-an-entity.md` | -| A2 | `cloud/akeneo` | "grouping product attributes into families and attribute groups" | Akeneo **families / attribute groups** docs | ⬜ to verify | -| A3 | `cloud/example-dam` | "digital asset review and approval workflow" | example-dam **Marketing Operations workflow** (e.g. `…/workflow_admin/workflow_designer_concepts…`) | ✅ verified — hit `Marketing_Operations_Help/workflow_admin/workflow_designer_concepts.html.md` | -| A4 | `cloud/bynder` | "set up an asset approval workflow and organize assets into collections" | Bynder **Asset-Workflow / collections** (e.g. `…/Asset-Workflow/…Asset-Workflow.md`) | ⬜ to verify (Asset-Workflow.md already appeared as a side hit under B1) | -| A5 | `cloud/digizuite` | "generate media renditions and publish them to a destination" | Digizuite **renditions / transformation / publishing** docs | ⬜ to verify | - -**Expected:** all five PASS. Record the top chunk `path` + `chunk_ref` for each in -the results table. - ---- - -## B. True negatives — a concept that lives in the *other* domain - -Take a concept a product genuinely **does not have** and query the product that -lacks it. Semantic search will still return *something* (it always ranks the -top-k), so PASS is defined by **off-topic** content, not an empty result. - -| # | Project | Query (from the *wrong* domain) | PASS = top chunks are OFF-topic (concept absent) | Result | -|---|---------|--------------------------------|--------------------------------------------------|--------| -| B1 | `cloud/bynder` (DAM) | "how are product entities linked to variants and sales channels" (PIM) | No PIM entity/link model; hits are generic DAM articles | ✅ **clean negative** — `Product-Feedback…`, `…AI-Agents…`, Studio; no PIM model | -| B2 | `cloud/example-dam` (DAM) | "how are product entities linked to variants and sales channels" (PIM) | No PIM entity model in a DAM/MO product | ✅ **clean negative** — `system_types_reference`, DAM `RecordLink` field, `clients_associated_programs`; no PIM entity/variant/channel model | -| B3 | `cloud/inriver` (PIM) | "automatically generate cropped image renditions and file derivatives from a master asset" (DAM) | No rendition/derivative engine in a PIM | ✅ **clean negative** — top hits are release notes / product announcements; inriver has no image-rendition transformation | -| B4 | `cloud/akeneo` (PIM) | "track marketing campaign budget spend and program financial actuals" (example-dam MO) | No budget/financials in a PIM catalog | ✅ **clean negative** — top hits are Google-Shopping insights / Studio analytics; Akeneo has no marketing-spend tracking | - -> **📌 Key lesson — how to design a clean true-negative probe.** -> A clean true negative needs a concept with **no adjacent feature** in the target -> product. Two examples of what *not* to do, found while building this scenario: -> - "brand guidelines portal" against `cloud/inriver` → matched inriver's own -> **Brand Store** (`…Introduction-to-the-new-Brand-Store.md`). -> - "workflow designer and task approvals" against `cloud/akeneo` → matched Akeneo's -> own **collaboration workflows** (`…what-are-collaboration-workflows.md`). -> -> Neither was the queried DAM/MO concept, but both are *genuine* features of the PIM -> product — so semantic search correctly surfaced the nearest real concept. That is -> **semantic search working**, not a scoping failure. The B3/B4 queries above were -> therefore sharpened to concepts that are truly unique to the *other* domain -> (rendition generation = DAM engine; marketing budget = example-dam MO), which produce -> clean negatives. Rule of thumb: probe with a **product-unique** concept, never a -> generic verb like "workflow" or "portal". -> -> A real regression would be a DAM index returning an **on-topic PIM entity-model** -> doc for B1/B2 — that would mean mis-scoped mounts or corpus contamination. - ---- - -## C. Cross-product overlap — shared concept, per-product answers - -A concept the three DAMs **all** share ("metadata fields on a digital asset"). -Query each DAM individually, then the whole peer via the group. - -| # | Scope | Query | PASS = | -|---|-------|-------|--------| -| C1 | `project=cloud/example-dam` | "add and edit metadata fields on a digital asset" | example-dam field/classification docs | -| C2 | `project=cloud/bynder` | "add and edit metadata fields on a digital asset" | Bynder metaproperty/tagging docs | -| C3 | `project=cloud/digizuite` | "add and edit metadata fields on a digital asset" | Digizuite metadata docs | -| C4 | `group=docs` | "add and edit metadata fields on a digital asset" | Fused results from **multiple** peers; each result carries the correct `source`/`chunk_ref` for its origin | - -**Expected:** C1–C3 each return that product's own vocabulary; C4 interleaves hits -from more than one DAM and every result is correctly attributed. (C4 also exercises -RRF fusion across federated peers.) - ---- - -## D. `get_chunk` namespaced round-trip — Stage A acceptance / regression - -This is the **gating** test for the fix. inriver on the peer is a multi-repo index, -which is exactly the shape that triggered the original `ambiguous_chunk_id` bug. - -**Steps** -1. `search(project="cloud/inriver", query="what is an entity", limit=3)` → note the top result's `chunk_ref`. -2. `get_chunk(chunk_ref="", context_lines=5)`. - -| Binary | Step 1 `chunk_ref` shape | Step 2 result | -|--------|--------------------------|---------------| -| **Old (pre-Stage-A, current live serve)** | legacy `"cloud:"` — alias dropped | ❌ FAIL — `ambiguous_chunk_id` (peer can't disambiguate the multi-repo index), the bug that started this | -| **New (Stage-A binary)** | namespaced `"cloud/inriver:"` | ✅ PASS — returns the chunk body; `project=inriver` is forwarded to the peer so the lookup is unambiguous | - -**PASS criteria (new binary):** -- `chunk_ref` is `"cloud/inriver:"` (namespaced). -- `get_chunk` returns the chunk `content` (the entity-definition prose), **not** an error. -- A legacy `"cloud:"` ref still resolves via the group-scope fallback (backward-compat) — optional extra check. - ---- - -## E. Fail-open sanity (web-guard interplay) — optional - -Confirms the guard doesn't over-block once mounts exist and steers correctly. - -| # | Setup | Action | PASS = | -|---|-------|--------|--------| -| E1 | mounts present (P1) | trigger a `WebSearch` on a product-doc question | web-guard **denies once** with guidance to `search(project="cloud/…")` + `get_chunk(chunk_ref=…)` | -| E2 | same query retried within 5 min | repeat the `WebSearch` | guard **allows** it (retry-escape) | -| E3 | `remote_mounts` empty in `repos.json` | trigger a `WebSearch` | guard **passes through** (fail-open, nothing to steer toward) | - ---- - -## F. Cross-vendor overlap + isolation (5 scenarios) - -Where **B** proved isolation (a concept absent from the wrong domain), **F** proves -the complementary half: a concept **shared** across vendors must surface hits from -**multiple vendors at once** via `group="docs"` (federated RRF fusion) — while a -domain-specific concept still stays absent from the other domain (isolation). - -Run each **overlap query** with `search(group="docs", …)` and confirm ≥2 vendors -return *on-topic* hits. Run each **isolation probe** with `project=""` -and confirm *off-topic* results. All rows below were executed (Run 1). - -> **How `group="docs"` fusion reads:** RRF interleaves each peer's rank-1 hit at the -> same top score (~0.0476), so a healthy overlap looks like *one strong hit per -> relevant vendor* stacked at the top. Judge by the `path`, not the score. - -### F1 — Category hierarchy *(cross-domain organizational concept)* -- **Overlap** `group=docs`: *"organize products into a category hierarchy or category tree"* -- **On-topic, multi-vendor:** akeneo `…/serenity-what-is-a-category.md`, bynder `…/Glossary/…What-is-a-Taxonomy.md`, digizuite `…/api/tree/nodes/item/…` -- **Adjacent (not wrong):** example-dam `expense_hierarchies` (MO financial), inriver release notes, custom-kb classification pickers -- **Verdict:** ✅ PASS — 3 vendors on-topic across **both** domains (PIM akeneo + DAM bynder/digizuite) - -### F2 — Product data completeness *(PIM-owned overlap + DAM isolation)* -- **Overlap** `group=docs`: *"measure product data completeness and enrichment quality"* -- **On-topic PIM:** inriver `…/working-in-enrich/…different-completeness-rules….md`, akeneo `…/understand-data-quality.md` -- **Isolation probe** `project=cloud/bynder`: top hits `Tips-For-Measuring-Success-And-Adoption`, `Stibo-Integration`, `Collections-Dashboard` — **no** product-completeness concept -- **Verdict:** ✅ PASS — two PIMs own it; a pure DAM does not (clean isolation) - -### F3 — Asset access permissions *(DAM-owned overlap)* -- **Overlap** `group=docs`: *"restrict who can view or download an asset using permissions and rights"* -- **On-topic:** bynder `…/Permission-Management/…Customize-User-Permissions-to-Download-Assets.md`, digizuite `…/api/assets/security/…`, example-dam `…/rights_reference.html.md`, akeneo (DAM module) `…/set-rights-on-your-asset-families.md` -- **Isolation signal** inriver: `…/entities/…Locking-Entities.md` — its own *entity-locking*, not asset download → stays in its lane -- **Verdict:** ✅ PASS — 3 DAMs + Akeneo's DAM module converge; PIM entity-locking is adjacent, not a false hit - -### F4 — Asset version history *(DAM-owned overlap + clean PIM isolation)* -- **Overlap** `group=docs`: *"keep version history of an asset and revert to a previous version"* -- **On-topic:** example-dam `…/digital_assets_creating_versions.html.md`, bynder `…/Upload/…Upload-New-Version-of-an-Asset.md`, digizuite `…/api/assets/create-versions.md`, akeneo `…/how-to-view-and-restore-a-previous-version-of-an-asset.md` -- **Isolation probe** `project=cloud/inriver`: **release notes only** — inriver (PIM) has no asset-versioning/revert -- **Verdict:** ✅ PASS — strongest 4-vendor DAM overlap + clean PIM isolation - -### F5 — Publish / syndicate to a channel *(true cross-domain overlap — the highlight)* -- **Overlap** `group=docs`: *"publish or syndicate content out to an external channel or destination"* -- **On-topic PIM:** inriver `…October-2025…Syndication-Workflows….md`, akeneo `…/managing-and-distributing-enhanced-content.md` -- **On-topic DAM:** example-dam `…/integration_workbench_publishers_concept.html.md`, bynder `…/Guide-to-Delivering-Multi-Channel-Content-with-Content-Workflow.md`, digizuite `…/api/admin/mediatranscode.md` -- **Verdict:** ✅ PASS — on-topic hits from **both** domains; the best single demonstration of full-peer federated fusion - -### Verdict — F (overlap + isolation) - -**5/5 PASS.** Federated RRF fusion surfaces the right *set* of vendors for a shared -concept (F1/F5 span both domains; F3/F4 converge the DAMs), and isolation still holds -where a concept is domain-specific (F2 PIM-only, F4 PIM has no asset versioning). This -is the positive counterpart to B: not just "not found in the wrong place", but -"found across all the right places, each correctly attributed". - ---- - -## Run 1 — executed results (Stage-A binary, serve restarted) - -`chunk_ref` came back **namespaced** (`cloud/:`) and `source` = `cloud/` -on every remote result → **P3 PASS**, the Stage-A fix is live. - -| Case | Scope | Query | Top chunk `path` | `chunk_ref` | Verdict | -|------|-------|-------|------------------|-------------|---------| -| A1 | cloud/inriver | product↔variant↔channel | `…/elastic-data-model…/What-is-an-entity.md` + `…/Intelligent-linking-of-Entities…md` | `cloud/inriver:1004` | ✅ PASS | -| A2 | cloud/akeneo | families/attribute groups | `…/serenity-what-is-a-family.md` + `…/manage-attribute-inheritance.md` | `cloud/akeneo:3770` | ✅ PASS | -| A3 | cloud/example-dam | asset review/approval | `…/workflow_admin/workflow_designer_concepts.html.md` | `cloud/example-dam:9645` | ✅ PASS | -| A4 | cloud/bynder | approval workflow + collections | `…/Asset-Workflow/…Asset-Workflow-Assets.md` + `…Asset-Workflow.md` | `cloud/bynder:1458` | ✅ PASS | -| A5 | cloud/digizuite | renditions + publish | `…/LegacyService/POST/api/renditions/_assetId_.md` | `cloud/digizuite:491` | ✅ PASS | -| B1 | cloud/bynder | PIM entity model (neg) | off-topic (Product-Feedback, AI-Agents) | — | ✅ PASS (clean neg) | -| B2 | cloud/example-dam | PIM entity model (neg) | off-topic (`system_types_reference`, DAM `RecordLink`) | — | ✅ PASS (clean neg) | -| B3 | cloud/inriver | DAM rendition/derivative engine (neg) | off-topic (release notes / product announcements) | — | ✅ PASS (clean neg) | -| B4 | cloud/akeneo | example-dam MO budget/financials (neg) | off-topic (Google-Shopping insights, Studio analytics) | — | ✅ PASS (clean neg) | -| C1 | cloud/example-dam | asset metadata fields | `…/Asset_Studio_Help/MetadataTemplates.htm.md` | `cloud/example-dam:5874` | ✅ PASS | -| C2 | cloud/bynder | asset metadata fields | `…/Upload/…Understanding-And-Using-Metadata.md` | `cloud/bynder:1116` | ✅ PASS | -| C3 | cloud/digizuite | asset metadata fields | `…/GET/api/metafield/asset-info.md` + `…/POST/api/metadata/editor.md` | `cloud/digizuite:1008` | ✅ PASS | -| C4 | group=docs | asset metadata fields | fused: digizuite + inriver + custom-kb + bynder + example-dam + akeneo | mixed, each correctly attributed | ✅ PASS (RRF fusion + attribution) | -| D | cloud/inriver | `get_chunk("cloud/inriver:1004")` | returned full "What is an entity?" body, **no `ambiguous_chunk_id`** | `cloud/inriver:1004` | ✅ **PASS (gating)** | -| E1–E3 | web-guard | — | — | — | ⬜ not run this pass | - -### Verdict — Run 1 - -- **A (recall): 5/5 PASS** — semantic search finds the right doc in the owning product even when the query wording differs from the docs. -- **B (isolation): 4/4 clean negatives** — no cross-domain contamination. (B3/B4 were sharpened to product-unique concepts after the first draft's generic probes matched the PIMs' own adjacent features — see the 📌 note; that was test-design, not a product bug.) -- **C (overlap/fusion): 4/4 PASS** — per-product answers are product-specific, and `group=docs` fuses all six peers with correct `source`/`chunk_ref` attribution. -- **D (Stage-A gating): PASS** — namespaced `chunk_ref` round-trips; the original `ambiguous_chunk_id` bug is fixed on the live binary. - -**Overall: PASS.** The remote-mount semantic search behaves as designed; the only -follow-up is refining the true-negative probes (B3/B4) to product-unique concepts. - -**Overall PASS criterion (for re-runs) =** all A PASS (recall) **and** B shows no -on-topic cross-domain hit (isolation) **and** D PASS on the Stage-A binary (round-trip). -C and E are supporting evidence. diff --git a/build.ps1 b/build.ps1 index 7caa7743..50acac0c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -10,6 +10,12 @@ Version bumping is handled by the pre-commit hook, NOT here. + Output is redirected to .tmp/build-.log in the repo and printed + afterwards, so a lingering cargo child process can never stall a live + pipe. If other cargo/rustc processes are already running, this script + WARNS but never kills them — they may belong to other sessions, and + cargo's own file lock will serialize the builds safely. + .EXAMPLE .\build.ps1 Builds in debug mode @@ -47,20 +53,46 @@ try { # run and surface its own error. } +# --- Orphaned-build detection: warn only, NEVER kill --- +# A cargo.exe/rustc.exe from a killed previous run can hold the target-dir +# build lock, making this build block forever on "Blocking waiting for file +# lock". Detect and report; the human decides what to do with them. They may +# perfectly well be legitimate builds from other sessions. +$orphans = @(Get-Process -Name cargo, rustc -ErrorAction SilentlyContinue) +if ($orphans.Count -gt 0) { + Write-Host " [warn] cargo/rustc already running (possible orphan holding the target-dir lock):" -ForegroundColor Yellow + foreach ($p in $orphans) { + Write-Host (" pid={0} name={1} started={2}" -f $p.Id, $p.ProcessName, $p.StartTime) -ForegroundColor Yellow + } + Write-Host " [warn] if this build hangs on 'Blocking waiting for file lock', terminate the stale process manually." -ForegroundColor Yellow +} + # Determine build mode $BuildMode = if ($Release) { "release" } else { "debug" } Write-Host "Building in $BuildMode mode..." -ForegroundColor Yellow +# --- Redirect output to a file in the repo, print afterwards --- +# Piping cargo's live output can hang indefinitely when a child process +# outlives the build (stale watchexec/rustc holding the pipe open). Writing +# to .tmp/build-.log and printing the finished file afterwards cannot +# stall: the file handle closes the moment cargo exits. +$TmpDir = Join-Path $ScriptDir ".tmp" +New-Item -ItemType Directory -Force -Path $TmpDir | Out-Null +$LogFile = Join-Path $TmpDir "build-$BuildMode.log" + if ($Release) { - & cargo build --release + & cargo build --release 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 } else { - & cargo build + & cargo build 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 } -if ($LASTEXITCODE -ne 0) { - Write-Host "Build failed!" -ForegroundColor Red - exit $LASTEXITCODE +$BuildExit = $LASTEXITCODE +Get-Content $LogFile | ForEach-Object { Write-Host $_ } + +if ($BuildExit -ne 0) { + Write-Host "Build failed! (full output: $LogFile)" -ForegroundColor Red + exit $BuildExit } Write-Host "Build completed: target/$BuildMode/codesearch.exe" -ForegroundColor Green diff --git a/build.rs b/build.rs index 53983a82..1cb5b807 100644 --- a/build.rs +++ b/build.rs @@ -37,11 +37,14 @@ fn main() { // Construct full version string let version_full = format!("{}+{}", cargo_version, commit_count); - // Set environment variables for the main binary - println!("cargo:rustc-env=DEMONGREP_VERSION_FULL={}", version_full); - println!("cargo:rustc-env=DEMONGREP_COMMIT_HASH={}", commit_hash); - println!("cargo:rustc-env=DEMONGREP_COMMIT_COUNT={}", commit_count); - println!("cargo:rustc-env=DEMONGREP_BRANCH={}", branch_name); + // Set environment variables for the main binary. Emitted under the + // CODESEARCH_ prefix so they are identifiable in build output as this + // project's provenance (the DEMONGREP_ prefix was a leftover from the + // project this fork started from; nothing read it). + println!("cargo:rustc-env=CODESEARCH_VERSION_FULL={}", version_full); + println!("cargo:rustc-env=CODESEARCH_COMMIT_HASH={}", commit_hash); + println!("cargo:rustc-env=CODESEARCH_COMMIT_COUNT={}", commit_count); + println!("cargo:rustc-env=CODESEARCH_BRANCH={}", branch_name); // Also set for display in --version output println!("cargo:rustc-env=CARGO_PKG_VERSION_FULL={}", version_full); diff --git a/docs/federated-silent-poll/worklog.md b/docs/federated-silent-poll/worklog.md deleted file mode 100644 index 49be0788..00000000 --- a/docs/federated-silent-poll/worklog.md +++ /dev/null @@ -1,187 +0,0 @@ -# Worklog — federated peer woken with no query behind it - -| | | -|---|---| -| **Branch** | `fix/federated-silent-poll-diagnosis` | -| **Base SHA** | `55fa36b` (🐛 fix: log keep-warm pings + warn when target isn't self) | -| **Scope** | Stop a scale-to-zero federated cloud peer being woken, and kept warm, with no federated query behind it. Local repo polling must stay untouched. | -| **Status** | **Shipped and verified in production.** Merged to `develop` via PR #192 (`b6cb48f`); deployed to the cloud peer as revision `codesearch-serve--0000024`. | -| **Latest test result** | CI green on PR #192 (test-linux, test-windows, csharp-integration-tests, CodeQL, Analyze). Locally: `cargo test --lib --bins` → **1134 passed, 42 ignored**; `cargo fmt --check` and `cargo clippy --all-targets -- -D warnings` clean | - -## The requirement - -Background polling of **local** repos is fine. Background polling of a -**federated peer** must never happen — an explicit design constraint from the -original federation design, restated by the user as: - -> "hij mag die repos pollen LOKAAL maar niet federated !!! dat had ik nochthans -> in de specs effectief gezegd bij het ontwerp" - -A peer staying warm for an hour *after real use* is **correct** and was -explicitly confirmed as such. The defect was the peer being **woken** with no -query, and then staying warm off that spurious wake. - -## Stage 0 — diagnosis (no commit) - -The pre-existing `DIAGNOSE_FEDERATED_KEEP_WARM.md` blamed a misconfigured local -`CODESEARCH_KEEP_WARM_URL`. **Disproven** on four independent grounds: the env -var is set nowhere locally (process env, HKCU, HKLM, every shell profile); the -one-time `🔥 keep-warm enabled` line appears in zero logs from 2026-04-26 on; -that absence is meaningful because `init_serve_logger` is always file-only in -serve mode and those logs do carry other INFO lines; and no local process held -a `:443` connection. `Watch-CodesearchServeReplicas.ps1` was also eliminated -(polls `/status` every 20s, but last ran 2026-07-05). - -Azure Log Analytics ground truth, over a window with **zero** federated -searches: wakes **120 / 121 / 120 minutes** apart, each warm period **~67 min**, -nightly sleeps exactly 2h00m30s apart → **≈13.4h warm/day, ~56% duty cycle**. -The 120-minute spacing is the tell: it is the **local** host's 2h default, not -any value configured on the peer. - -Two defects, one triggering and one amplifying: - -1. **Trigger** — the TUI's `spawn_remote_discovery` used - `state.idle_suspend_secs()` as a baseline poll interval and ran a `JoinSet` - `/status` fan-out to every peer. The poll *itself* was the ingress traffic. -2. **Amplifier** — keep-warm's `most_recent_tool_call().unwrap_or(start)`. - `/status` and `/healthz` never call `record_tool_call`, so any non-tool-call - wake self-warmed for the full idle window. Unreachable in the case it was - written for, so its only practical effect was rewarding spurious wakes - (~11×). - -## Stage 1 — remove the federated timer poll - -- **Commit:** `6f1d1c5` · **Review:** ⚠️ PASS WITH REMARKS (round 1) → - fixes amended → **PASS, zero code defects** (round 2, cap reached). -- `spawn_remote_discovery` no longer polls on any cadence; the tick is - **config-only** (`REMOTE_ROW_REFRESH_SECS` = 5s, zero HTTP) so mount/unmount - edits and `l` reloads still surface. Contact is activity-poke (single peer, - never a fan-out) or the `i` keypress only. -- Removed `ServeState::idle_suspend_secs` (field, env init, getter, - `--idle-suspend-secs` override) — write-only once the cadence went. - Keep-warm resolves flag > env > default itself, so the flag still works. -- Round-1 fixes amended in: `tui_common.rs` `activity_stale` doc; and the - snapshot emit, previously gated on a non-empty peer list, which left rows on - screen forever after the last peer was removed. - -**Files:** `src/constants.rs`, `src/serve/mod.rs`, `src/serve/tui.rs`, -`src/serve/tui_common.rs` - -## Stage 2 — keep-warm requires a real tool call - -- **Commit:** `12edcf2` · **Review:** ✅ **PASS, zero findings.** -- The `unwrap_or(start)` fallback is gone; with no recorded tool call the loop - does not ping and lets the host suspend the replica. -- Reviewer independently confirmed warm-after-real-use does **not** regress: an - inbound federated search forces `project=`, reaching - `record_tool_call`; and `last_tool_call` is **insert-only** (no - `remove`/`clear`/`retain`, untouched by repo idle-eviction), so once one real - query lands the old behaviour holds for the process lifetime. -- Also fixed the `55fa36b` startup warning, which false-positived on the only - correct deployment: Azure binds `0.0.0.0` while the target is the ingress - FQDN. Wildcard bind ⇒ external host unknown ⇒ stay silent. Rule extracted to - the testable `keep_warm_foreign_target` helper (5 new tests). - -**Files:** `src/serve/mod.rs`, `src/serve/tests.rs` - -## Stage 3 — documentation - -- **Commit:** `3bcf153` · **Review:** ✅ **PASS** (final full-branch review, - `55fa36b...3bcf153`) — "Nothing further is owed on this branch." -- `DIAGNOSE_FEDERATED_KEEP_WARM.md` rewritten (moved from `docs/`): confirmed - root cause, ground truth, the violated requirement, and the **rejected - reasoning** recorded so it is not re-litigated a fourth time. -- `AGENTS.md` bullet rewritten as an explicit design constraint; it had - asserted the removed cadence as current and named a deleted field — the very - mechanism by which this behaviour was re-introduced twice. -- `CHANGELOG.md`: fix entry under `[1.2.4] (unreleased)`. An earlier draft had - put it under `[1.2.0]` — a **real tag** that shipped #181/#184; the reviewer - caught this, and both original entries were restored **verbatim** (confirmed - by diff) and marked superseded. -- `README.md`: grep-guard bullet corrected to the `/healthz` liveness probe. - -**Files:** `AGENTS.md`, `CHANGELOG.md`, `README.md`, -`DIAGNOSE_FEDERATED_KEEP_WARM.md` *(new)*, `docs/diagnose-federated-keep-warm.md` *(deleted)* - -## Stage 4 — the branch had no CI at all - -- **Commit:** `4add0d1` · **Review:** covered by the final full-branch review. -- While preparing the PR it turned out `ci.yml`'s push trigger is a **prefix - allowlist** that did not include `fix/**`. Every `fix/...` branch — the - repo's own documented naming convention — had therefore merged into - `develop` without ever running fmt, clippy or a single test. The PR still - showed green because CodeQL is a separate `pull_request`-triggered workflow - and was the only check present. -- Added `fix/**`, plus a comment explaining the footgun and how to verify - (`gh pr checks ` must list the CI jobs, not just CodeQL). -- Proven by self-test: `08276de` → CodeQL only; `4add0d1` → CI + CodeQL. -- `chore/**` was added later, in PR #193. - -**Files:** `.github/workflows/ci.yml` - -## Stage 5 — deployment and production verification - -- **Merged:** PR #192 → `develop` (`b6cb48f`), auto-bumped to 1.2.5. -- **Local instance:** deployed by the user via `copy-to-common`. This carries - Defect 1 (the TUI timer poll), which only ever ran on the *local* side — so - the trigger was removed first. -- **Cloud peer:** image `codesearch-serve:8d7261e6d` built from a clean - `git archive` of `develop` and deployed as revision - `codesearch-serve--0000024`. Config verified intact across the update: 12 - env vars, 4 secretRefs, `CODESEARCH_IDLE_SUSPEND_SECS=1800`. `/healthz` - returned 200 in 147 ms. Registry size 160,647,888 B vs 160,629,714 B for the - previous image — an 18 KB difference, so no size regression. -- **Idle window** was separately reduced 3600 → 1800 s at the user's request, - halving the cost of any wake that does still occur. -- **Verified:** the replica scaled to 0 about 10 minutes after deploy and - **stayed at 0 across six consecutive one-minute checks with no traffic**. - This is positive evidence rather than mere absence of symptoms: under the old - binary `most_recent_tool_call()` would have been `None`, fallen back to the - process start time, and self-pinged every 120 s for the full 30-minute idle - window — reaching 0 at 10 minutes was not possible. - -**Build note:** two `az acr build` runs failed at the identical step -(`COPY --from=builder /models.tar.gz`) with -`failed to export image: ... layer does not exist`. Layer digests differed -between runs, so it was not a poisoned cache; the Dockerfile is byte-identical -to the one that built the previously deployed image. Root cause sits in the ACR -Tasks build agent (registry is Basic SKU), not in this repo. A local -`docker build` + `docker push` succeeded first time and was used instead. - -## Why this took three attempts across three PRs - -PR #181 introduced the cadence on the reasoning that polling no faster than the -host's suspend term is harmless; PR #184 named "waking the scale-to-zero cloud -peer for no real reason" as the defect and then explicitly sanctioned that same -cadence. The flaw: **not keeping a peer awake past its suspend term is strictly -weaker than not waking it**, and the two windows were unrelated values anyway -(local host vs. remote peer). Both PRs correctly said "local repos unaffected", -confirming the local/federated split was real — but honoured it in only one -direction. - -## Open follow-ups - -- **Residual, known and not currently exploitable:** the MCP `status` **tool**, - when project-scoped, *does* record a tool call (`allow_unscoped=true` reduces - the guard to `!is_multi`), so an automated poller of that tool would still buy - a warm window. No such poller exists — both `Watch-CodesearchServeReplicas.ps1` - and `FederationClient::list_repos` use the HTTP `/status` endpoint, which does - not record. First place to look if the symptom recurs. -- **Verified in production** (see Stage 5) over a ~6-minute window. Worth - re-running the original Log Analytics query over a **full day** to confirm the - duty cycle: was ≈13.4 h warm/day (~56%) at zero searches, expected now ≈0 with - wakes only behind real federated queries. The short window proves the - self-ping is gone; only a 24 h sample proves nothing else wakes it. -- **ACR Tasks cannot currently build this image** (Stage 5 build note). The - local `docker build` path works, but a CI/automated deploy would hit the same - failure. Worth a look before anyone automates the cloud deploy. - -## Security note - -None. No auth, network-exposure or data-handling surface changed; the branch -strictly *reduces* outbound traffic. - - diff --git a/docs/watcher-reindex-tui-visibility/worklog.md b/docs/watcher-reindex-tui-visibility/worklog.md deleted file mode 100644 index 61cfb85a..00000000 --- a/docs/watcher-reindex-tui-visibility/worklog.md +++ /dev/null @@ -1,87 +0,0 @@ -# Worklog — watcher reindex / TUI visibility - -- **Branch:** `fix/watcher-reindex-tui-visibility` -- **Base SHA:** `09b451aaa7372285f76d50fad71a035aa40e4fd5` (develop) -- **Scope:** Make watcher-triggered reindexes visible in the serve TUI and rebuild - C#/TypeScript symbols on branch switch. Three user-reported gaps: - A) branch switch never rebuilds symbols (find_impact goes stale); - B) the C# indicator never shows "Indexing" during a watcher rebuild; - C) symbol-rebuild log lines lack a repo label; - plus (gap #1) ordinary text-batch reindexes never signal "Indexing" in the TUI. -- **Status:** ✅ COMPLETE — all 3 stages + DRY refactor committed; every per-stage - review and the final full-branch review PASSED. Not pushed (awaiting user). -- **Latest test result:** `cargo fmt --all --check`, `cargo check --all-targets`, - `cargo clippy --all-targets -- -D warnings` clean; 609 lib tests pass. -- **Final review:** ✅ PASS on full diff `09b451a..78c9310` (holistic signal/label - balance, single-source-of-truth rebuild helper, no stale 2-arg notifier sites). - -## Root cause (from code, not memory) - -| Watcher path | Signalled `indexing_cb` (general TUI "Indexing")? | Set `CSharpIndexStatus::Indexing`? | Rebuilt symbols? | -|---|---|---|---| -| Text batch flush (`process_batch_with_stores`) | ❌ no (gap #1) | n/a | n/a | -| Branch switch | ✅ yes (text refresh only) | ❌ no | ❌ no — discards `.cs/.ts` buffers (gap A) | -| `.cs` debounce | ✅ yes | ❌ no (gap B) | ✅ yes (incremental) | -| `.ts` debounce | ✅ yes | n/a (no TS notifier) | ✅ yes (full) | - -The serve-layer helper `trigger_symbol_rebuild` (src/serve/mod.rs) already sets -`CSharpIndexStatus::Indexing` + `begin_indexing` + Full rebuild, but the watcher -in `IndexManager` cannot reach it — it only holds the two callbacks. - -## Stages - -### Stage 1/3 — Text-batch TUI visibility + repo-label logging (gap #1 + Fix C text paths) -- Commit: `3d43993da4d40d4711a6bb9d443bee3019e21ffe` — review: ✅ PASS (no remarks). -- Wrapped the FSW text-batch flush in `indexing_cb(true/false)` so ordinary file - edits surface as "Indexing" in the TUI (the `IndexingStatusCallback` doc already - claimed it fired on "batch flushes"; it never did). -- Added a `repo_label` (repo directory name = serve alias) to the watcher task and - interpolated it into batch-flush and branch-change log lines. -- Files: `src/index/manager.rs`. - -### Stage 2/3 — C# indicator shows "Indexing" during watcher rebuild (Fix B) -- Commit: `2dbafa3d0b8cd4f6996267b40c2c6806cb769121` — review: ✅ PASS (no remarks). -- Refactored `CSharpRebuildNotifier` from `Fn(bool, Option)` to a 3-state - `SymbolRebuildSignal { Started, Succeeded, Failed(String) }`. The watcher emits - `Started` right after the applies/available gate, so `make_csharp_notifier` sets - `CSharpIndexStatus::Indexing` for the rebuild duration (was Ready/Error only). -- Guards (`!applies_to`, `!is_available`) return BEFORE `Started`, so the indicator - is never left stuck on `Indexing`. -- Also labelled every C# rebuild log line with `repo_label`; refreshed two stale - callback doc comments. -- Files: `src/index/manager.rs` (new file: no), `src/index/mod.rs`, `src/serve/mod.rs`. - -### Stage 3/3 — Branch-switch symbol rebuild (Fix A) -- Commits: `928273d6ee0e66f6f66f64fdcc8126a2e063919b` (feature) — - review: ⚠️ PASS WITH REMARKS (1 Important: duplicated full-rebuild block); - `a5f66c819d45bf0c9d49d74293ee299e5f06f9e9` (remark fix) — extracted - `IndexManager::run_full_rebuild_logged`; re-review ⚠️ PASS WITH REMARKS - (one 4th copy left in the `.ts` path); `78c9310aa5f45b452d0929a586a96b7fb8a4b6cc` - (fold-in) — routed the `.ts` debounce rebuild through the same helper → - single source of truth for all four full-rebuild paths. 609 lib tests pass. -- Added `IndexManager::spawn_branch_change_symbol_rebuild(...)`: after the - branch-change text refresh, a fire-and-forget `spawn_blocking` runs a - `RebuildScope::Full` rebuild for every applicable+available language (C# + TS). - Full scope is correct — a branch switch rewrites arbitrary files, so no - incremental scope can be computed. -- Toggles the general `indexing_cb` label around the whole rebuild (only when a - language actually applies → no TUI flash otherwise); C# also drives the - `SymbolRebuildSignal` indicator. -- Files: `src/index/manager.rs`. - -## Follow-ups / notes -- **Deletions-only `.cs` debounce bug (pre-existing, found in Stage 2 review):** - when only `.cs` deletions are buffered (no modifications), the debounce path - builds empty `groups`/`ungrouped`, skips both the fallback and the per-group - loop, and emits `Started`→`Succeeded` WITHOUT running any rebuild — so the - forwarded `cs_deleted` set is never purged from LMDB and deleted symbols - linger. `manager.rs` grouped `.cs` path. Not fixed here (out of scope); a Full - rebuild (or `Files{changed:[], deleted}`) when `groups.is_empty() && !cs_deleted.is_empty()` - would fix it. Branch-switch deletions ARE now handled (Stage 3 Full rebuild). -- TypeScript watcher path updates no TUI symbol status (no TS notifier). Out of - scope this iteration; candidate follow-up. -- Watcher symbol-rebuild paths have no per-repo mutex guard (already a tracked - follow-up); concurrent rebuilds on the same repo are benign (alias-keyed). - Rapid successive branch switches could overlap Full rebuilds — same tradeoff. - - diff --git a/examples/benchmark_models.rs b/examples/benchmark_models.rs index da2dafb3..6df257bc 100644 --- a/examples/benchmark_models.rs +++ b/examples/benchmark_models.rs @@ -48,7 +48,7 @@ struct BenchmarkResult { fn main() -> Result<()> { println!("╔══════════════════════════════════════════════════════════════╗"); - println!("║ DEMONGREP EMBEDDING MODEL BENCHMARK ║"); + println!("║ CODESEARCH EMBEDDING MODEL BENCHMARK ║"); println!("╚══════════════════════════════════════════════════════════════╝"); println!(); diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index a4009e4e..e8539efc 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -8,8 +8,12 @@ namespace ScipCsharp; /// CLI entrypoint for scip-csharp. /// /// Subcommands: -/// index — compile solution, collect definitions, write SCIP JSON (fast, no FindReferencesAsync) -/// find-refs — resolve references for a single symbol on demand (for lazy find_impact caching) +/// index — compile solution, collect definitions, write SCIP JSON (fast, no FindReferencesAsync) +/// find-refs — resolve references for a single symbol on demand (for lazy find_impact caching) +/// batch-find-refs — resolve multiple symbols in one workspace session +/// serve — resident mode: load the workspace once, answer find-refs/reload +/// requests as JSON lines on stdin/stdout (todo #115; the Rust host +/// kills the process for teardown) /// public static class Program { @@ -44,10 +48,83 @@ public static async Task Main(string[] args) "index" => await RunIndexAsync(args[1..]).ConfigureAwait(false), "find-refs" => await RunFindRefsAsync(args[1..]).ConfigureAwait(false), "batch-find-refs" => await RunBatchFindRefsAsync(args[1..]).ConfigureAwait(false), + "serve" => await RunServeAsync(args[1..]).ConfigureAwait(false), _ => await UnknownCommand(args[0]).ConfigureAwait(false), }; } + /// + /// Solution path the serve loop was started with. Remembered so a + /// "reload" request without an explicit path re-opens the same solution. + /// Only meaningful in serve mode (resident); other subcommands are + /// single-shot processes. + /// + internal static string? CurrentServeSolution { get; private set; } + + // ── serve subcommand (resident mode) ───────────────────────────── + + private static async Task RunServeAsync(string[] args) + { + var parsed = ParseServeArgs(args); + if (parsed is null) return 1; + + if (!TryRegisterMsBuild(out var regErr)) { await Console.Error.WriteLineAsync(regErr).ConfigureAwait(false); return 1; } + + using var workspace = CreateTolerantWorkspace(); + + try + { + Console.Error.WriteLine($"serve: loading solution: {parsed}"); + await OpenSolutionFilteredAsync(workspace, parsed).ConfigureAwait(false); + } + catch (Exception ex) + { + await Console.Error.WriteLineAsync( + $"serve: [WARN] Solution load partially failed ({ex.GetType().Name}: {ex.Message}); " + + $"continuing with {workspace.CurrentSolution.Projects.Count()} loaded project(s).") + .ConfigureAwait(false); + + if (!workspace.CurrentSolution.Projects.Any()) + { + await Console.Error.WriteLineAsync( + $"serve: no projects loaded — cannot serve. Full error:{Environment.NewLine}{ex.StackTrace}") + .ConfigureAwait(false); + return 1; + } + } + + CurrentServeSolution = parsed; + + // The loop owns the process lifetime from here: stdin EOF or a + // "shutdown" request exits 0. The Rust host kills the process for + // teardown — disposing the Roslyn workspace reliably is not possible, + // process death is (todo #115). + return await ServeHost.LoopAsync( + workspace, + solution => OpenSolutionFilteredAsync(workspace, solution)).ConfigureAwait(false); + } + + private static string? ParseServeArgs(string[] args) + { + string? solutionPath = null; + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--solution": + solutionPath = RequireValidPath(args, ref i, "--solution", mustExist: true); + if (solutionPath is null) return null; + break; + default: + Console.Error.WriteLine($"Unknown argument: {args[i]}"); + return null; + } + } + + if (string.IsNullOrEmpty(solutionPath)) { Console.Error.WriteLine("serve: --solution is required"); return null; } + return solutionPath; + } + // ── index subcommand ───────────────────────────────────────────── private static async Task RunIndexAsync(string[] args) @@ -780,6 +857,7 @@ private static void PrintUsage() Console.WriteLine(" scip-csharp find-refs --solution --symbol --output "); Console.WriteLine(" scip-csharp batch-find-refs --solution --symbols-file --output "); Console.WriteLine(" scip-csharp batch-find-refs --solution --symbols --output "); + Console.WriteLine(" scip-csharp serve --solution "); Console.WriteLine(); Console.WriteLine("Options (index):"); Console.WriteLine(" --solution Path to .sln file"); @@ -798,12 +876,17 @@ private static void PrintUsage() Console.WriteLine(" --symbols-file File with one SCIP key per line"); Console.WriteLine(" --symbols Semicolon-separated SCIP keys"); Console.WriteLine(" --output Output JSON file path"); + Console.WriteLine(); + Console.WriteLine("Options (serve):"); + Console.WriteLine(" --solution Path to .sln file (loaded once; the process then"); + Console.WriteLine(" serves find-refs/reload requests as JSON lines on"); + Console.WriteLine(" stdin/stdout until EOF or a shutdown request)"); } [ExcludeFromCodeCoverage] private static async Task UnknownCommand(string cmd) { - await Console.Error.WriteLineAsync($"Unknown command: '{cmd}'. Use 'index', 'find-refs', or 'batch-find-refs'.").ConfigureAwait(false); + await Console.Error.WriteLineAsync($"Unknown command: '{cmd}'. Use 'index', 'find-refs', 'batch-find-refs', or 'serve'.").ConfigureAwait(false); return 1; } } diff --git a/helpers/csharp/ReferenceResolver.cs b/helpers/csharp/ReferenceResolver.cs index 6b68bb88..65a96e0a 100644 --- a/helpers/csharp/ReferenceResolver.cs +++ b/helpers/csharp/ReferenceResolver.cs @@ -22,7 +22,8 @@ public async Task FindRefsAsync(Solution solution, string scipKe { var output = new FindRefsOutput { Symbol = scipKey }; - var (symbolMap, projectRoot) = await BuildSymbolMapAsync(solution).ConfigureAwait(false); + var (symbolMap, projectRoot, mapWarnings) = await BuildSymbolMapAsync(solution).ConfigureAwait(false); + output.Warnings.AddRange(mapWarnings); var targetSymbol = FindSymbolByKey(symbolMap, scipKey); if (targetSymbol is null) @@ -33,8 +34,9 @@ public async Task FindRefsAsync(Solution solution, string scipKe Console.Error.WriteLine($"find-refs: resolving references for {scipKey}..."); - var refs = await ResolveReferencesAsync(targetSymbol, solution, projectRoot).ConfigureAwait(false); + var (refs, warnings) = await ResolveReferencesAsync(targetSymbol, solution, projectRoot).ConfigureAwait(false); output.References.AddRange(refs); + output.Warnings.AddRange(warnings); Console.Error.WriteLine($"find-refs: found {output.References.Count} reference(s)"); return output; @@ -48,7 +50,7 @@ public async Task BatchFindRefsAsync(Solution solution, IRe { var results = new BatchFindRefsOutput(); - var (symbolMap, projectRoot) = await BuildSymbolMapAsync(solution).ConfigureAwait(false); + var (symbolMap, projectRoot, mapWarnings) = await BuildSymbolMapAsync(solution).ConfigureAwait(false); // Build reverse map: scip_key → ISymbol for O(1) lookup var keyToSymbol = new Dictionary(); @@ -63,6 +65,9 @@ public async Task BatchFindRefsAsync(Solution solution, IRe { var scipKey = scipKeys[i]; var output = new FindRefsOutput { Symbol = scipKey }; + // Map-level warnings apply to every symbol: a project that failed + // to compile hides its symbols from every resolution. + output.Warnings.AddRange(mapWarnings); if (!keyToSymbol.TryGetValue(scipKey, out var targetSymbol)) { @@ -71,8 +76,9 @@ public async Task BatchFindRefsAsync(Solution solution, IRe continue; } - var refs = await ResolveReferencesAsync(targetSymbol, solution, projectRoot).ConfigureAwait(false); + var (refs, warnings) = await ResolveReferencesAsync(targetSymbol, solution, projectRoot).ConfigureAwait(false); output.References.AddRange(refs); + output.Warnings.AddRange(warnings); Console.Error.WriteLine($"batch-find-refs: [{i + 1}/{scipKeys.Count}] {scipKey} → {refs.Count} ref(s)"); results.Results.Add(output); @@ -85,11 +91,14 @@ public async Task BatchFindRefsAsync(Solution solution, IRe /// /// Builds the symbol map by compiling all projects in the solution. - /// Returns the map and the common project root for relative path computation. + /// Returns the map, the common project root for relative path computation, + /// and one warning per project that failed to compile — those projects' + /// symbols are silently missing from the map, so the caller must be told. /// - private async Task<(Dictionary SymbolMap, string? ProjectRoot)> BuildSymbolMapAsync(Solution solution) + private async Task<(Dictionary SymbolMap, string? ProjectRoot, List Warnings)> BuildSymbolMapAsync(Solution solution) { var symbolMap = new Dictionary(SymbolEqualityComparer.Default); + var warnings = new List(); Console.Error.WriteLine("find-refs: building symbol map from solution..."); foreach (var project in solution.Projects) @@ -98,6 +107,7 @@ public async Task BatchFindRefsAsync(Solution solution, IRe if (compilation is null) { Console.Error.WriteLine($"[WARN] find-refs: could not compile {project.Name}"); + warnings.Add($"could not compile project '{project.Name}' — its symbols are missing from the map"); continue; } SymbolIndexer.CollectSymbols(compilation.GlobalNamespace, symbolMap); @@ -110,7 +120,7 @@ public async Task BatchFindRefsAsync(Solution solution, IRe .Where(p => p is not null) .Cast()); - return (symbolMap, projectRoot); + return (symbolMap, projectRoot, warnings); } private static ISymbol? FindSymbolByKey(Dictionary symbolMap, string scipKey) @@ -123,10 +133,16 @@ public async Task BatchFindRefsAsync(Solution solution, IRe return null; } - private static async Task> ResolveReferencesAsync( + /// + /// Resolves references for one symbol. Returns the occurrences plus a + /// warning per survived failure — a caught FindReferencesAsync exception + /// must degrade the answer's honesty, not just its size. + /// + private static async Task<(List Refs, List Warnings)> ResolveReferencesAsync( ISymbol targetSymbol, Solution solution, string? projectRoot) { var results = new List(); + var warnings = new List(); try { @@ -157,8 +173,11 @@ private static async Task> ResolveReferencesAsync( Console.Error.WriteLine( $"[WARN] FindReferencesAsync failed for {targetSymbol.Name}: " + $"{ex.GetType().Name}: {ex.Message}"); + warnings.Add( + $"FindReferencesAsync failed for {targetSymbol.Name}: " + + $"{ex.GetType().Name}: {ex.Message}"); } - return results; + return (results, warnings); } } diff --git a/helpers/csharp/ScipModels.cs b/helpers/csharp/ScipModels.cs index f3e82243..973ad177 100644 --- a/helpers/csharp/ScipModels.cs +++ b/helpers/csharp/ScipModels.cs @@ -12,7 +12,7 @@ public sealed class ScipIndex public sealed class ScipMetadata { - public string Version { get; init; } = "1.0"; + public string Version { get; init; } = "2.0"; public string ToolInfo { get; init; } = "scip-csharp"; } @@ -43,9 +43,15 @@ public sealed class ScipSymbolInfo /// public sealed class FindRefsOutput { - public string Version { get; init; } = "1.0"; + public string Version { get; init; } = "2.0"; public string Symbol { get; set; } = ""; public List References { get; init; } = []; + /// + /// Non-fatal problems survived while resolving this symbol (a project + /// that failed to compile, a FindReferencesAsync exception). Non-empty + /// means the reference list may be incomplete. + /// + public List Warnings { get; init; } = []; } public sealed class FindRefsOccurrence @@ -62,6 +68,6 @@ public sealed class FindRefsOccurrence /// public sealed class BatchFindRefsOutput { - public string Version { get; init; } = "1.0"; + public string Version { get; init; } = "2.0"; public List Results { get; init; } = []; } diff --git a/helpers/csharp/ServeHost.cs b/helpers/csharp/ServeHost.cs new file mode 100644 index 00000000..0a23dbe2 --- /dev/null +++ b/helpers/csharp/ServeHost.cs @@ -0,0 +1,166 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.CodeAnalysis.MSBuild; + +namespace ScipCsharp; + +/// +/// Protocol models for the `serve` subcommand (resident mode). +/// +/// Line-based JSON over stdin/stdout; all logging stays on stderr so stdout +/// carries ONLY protocol responses. Requests are processed strictly +/// sequentially — one request at a time per workspace, by design (see todo +/// #115): a reload or shutdown must never race a find-refs. +/// +public sealed class ServeRequest +{ + /// "ping" | "find-refs" | "reload" | "shutdown". + public string Op { get; set; } = ""; + /// SCIP symbol key (find-refs only). + public string Symbol { get; set; } = ""; + /// Solution path (reload only; empty keeps the current one). + public string Solution { get; set; } = ""; +} + +public sealed class ServeResponse +{ + public bool Ok { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; set; } + /// True on the initial ready line, after the workspace is loaded. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Ready { get; set; } + /// Loaded project count (load/reload responses). + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Projects { get; set; } + /// find-refs payload — identical shape to the file the + /// `find-refs` subcommand writes, so the Rust host reuses one model. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public FindRefsOutput? Result { get; set; } +} + +/// +/// The resident request loop. Owns NOTHING about solution loading — Program +/// loads the workspace (same tolerant pipeline as every subcommand) and hands +/// it over together with a reload callback, so reload semantics stay defined +/// in exactly one place (OpenSolutionFilteredAsync). +/// +public static class ServeHost +{ + private static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + WriteIndented = false, + }; + + /// + /// Runs until stdin EOF or a "shutdown" request. Returns 0 on both — + /// the Rust side kills the process for teardown, which is the only + /// reliable way to dispose a Roslyn workspace. + /// + public static async Task LoopAsync( + MSBuildWorkspace workspace, + Func reloadSolution) + { + var resolver = new ReferenceResolver(); + + // Handshake: the Rust host waits for this line before sending any + // request — workspace load takes minutes and must not look like a + // hung helper. + await RespondAsync(new ServeResponse { Ok = true, Ready = true, Projects = workspace.CurrentSolution.Projects.Count() }) + .ConfigureAwait(false); + + string? line; + while ((line = await Console.In.ReadLineAsync().ConfigureAwait(false)) is not null) + { + var response = await HandleLineAsync(workspace, reloadSolution, resolver, line).ConfigureAwait(false); + if (response is null) + { + // shutdown — acknowledge, then exit the loop normally. + await RespondAsync(new ServeResponse { Ok = true }).ConfigureAwait(false); + return 0; + } + await RespondAsync(response).ConfigureAwait(false); + } + + // stdin EOF: the host closed the pipe (kill/teardown path) — exit 0. + return 0; + } + + /// Dispatch one request line. Returns null for "shutdown". + private static async Task HandleLineAsync( + MSBuildWorkspace workspace, + Func reloadSolution, + ReferenceResolver resolver, + string line) + { + ServeRequest? request; + try + { + request = JsonSerializer.Deserialize(line, Options); + } + catch (JsonException ex) + { + return new ServeResponse { Ok = false, Error = $"bad request: {ex.Message}" }; + } + + if (request is null || string.IsNullOrEmpty(request.Op)) + { + return new ServeResponse { Ok = false, Error = "missing op" }; + } + + try + { + switch (request.Op) + { + case "ping": + return new ServeResponse { Ok = true, Projects = workspace.CurrentSolution.Projects.Count() }; + case "find-refs": + return await HandleFindRefsAsync(resolver, workspace, request.Symbol).ConfigureAwait(false); + case "reload": + return await HandleReloadAsync(workspace, reloadSolution, request.Solution).ConfigureAwait(false); + case "shutdown": + return null; + default: + return new ServeResponse { Ok = false, Error = $"unknown op '{request.Op}'" }; + } + } + catch (Exception ex) + { + // One failed request must never kill the loop — the host decides + // lifecycle (kill on teardown), the helper answers on failures. + return new ServeResponse { Ok = false, Error = $"{ex.GetType().Name}: {ex.Message}" }; + } + } + + private static async Task HandleFindRefsAsync( + ReferenceResolver resolver, MSBuildWorkspace workspace, string symbol) + { + if (string.IsNullOrWhiteSpace(symbol)) + { + return new ServeResponse { Ok = false, Error = "find-refs requires symbol" }; + } + var result = await resolver.FindRefsAsync(workspace.CurrentSolution, symbol).ConfigureAwait(false); + return new ServeResponse { Ok = true, Result = result }; + } + + private static async Task HandleReloadAsync( + MSBuildWorkspace workspace, Func reloadSolution, string solution) + { + workspace.CloseSolution(); + var target = string.IsNullOrWhiteSpace(solution) ? Program.CurrentServeSolution : solution; + if (string.IsNullOrWhiteSpace(target)) + { + workspace.CloseSolution(); + return new ServeResponse { Ok = false, Error = "reload requires solution (none loaded)" }; + } + await reloadSolution(target).ConfigureAwait(false); + return new ServeResponse { Ok = true, Projects = workspace.CurrentSolution.Projects.Count() }; + } + + private static async Task RespondAsync(ServeResponse response) + { + await Console.Out.WriteLineAsync(JsonSerializer.Serialize(response, Options)).ConfigureAwait(false); + await Console.Out.FlushAsync().ConfigureAwait(false); + } +} diff --git a/helpers/csharp/SymbolIndexer.cs b/helpers/csharp/SymbolIndexer.cs index 6b4822af..8a49e4e9 100644 --- a/helpers/csharp/SymbolIndexer.cs +++ b/helpers/csharp/SymbolIndexer.cs @@ -218,7 +218,12 @@ internal static void CollectTypeSymbols(INamedTypeSymbol type, Dictionary /// Converts a Roslyn symbol to a SCIP-style symbol name. - /// Format: csharp <namespace> . <Type>#<member>(<params>). + /// Format: csharp <namespace> . <ContainingTypePath>#<member>(<params>). + /// Distinctness guarantees (see KeyFormatTests): generic arity is kept + /// (Foo`1 vs Foo, M`1 vs M), nested types carry their full containing-type + /// chain (Outer1.Inner vs Outer2.Inner) and parameter types are fully + /// qualified (A.P vs B.P). Any change here must bump the index version + /// (ScipModels "2.0") and SCIP_KEY_FORMAT so old indexes rebuild. /// internal static string SymbolToScipName(ISymbol symbol) { @@ -227,9 +232,10 @@ internal static string SymbolToScipName(ISymbol symbol) var ns = type.ContainingNamespace?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); if (ns?.StartsWith("global::") == true) ns = ns["global::".Length..]; + var typePath = ContainingTypePath(type); if (string.IsNullOrEmpty(ns)) - return $"csharp . . {type.Name}#"; - return $"csharp {ns} . {type.Name}#"; + return $"csharp . . {typePath}#"; + return $"csharp {ns} . {typePath}#"; } var containingType = symbol.ContainingType; @@ -240,14 +246,18 @@ internal static string SymbolToScipName(ISymbol symbol) if (typeNs?.StartsWith("global::") == true) typeNs = typeNs["global::".Length..]; - var typeName = containingType.Name; + var containingPath = ContainingTypePath(containingType); var prefix = string.IsNullOrEmpty(typeNs) - ? $"csharp . . {typeName}#" - : $"csharp {typeNs} . {typeName}#"; + ? $"csharp . . {containingPath}#" + : $"csharp {typeNs} . {containingPath}#"; return symbol switch { - IMethodSymbol method => $"{prefix}{method.Name}({FormatParameters(method.Parameters)}).", + // Arity on the method name (M`1) keeps void M() and void M() + // on distinct keys. + IMethodSymbol method => method.Arity > 0 + ? $"{prefix}{method.Name}`{method.Arity}({FormatParameters(method.Parameters)})." + : $"{prefix}{method.Name}({FormatParameters(method.Parameters)}).", IPropertySymbol prop => $"{prefix}{prop.Name}", IFieldSymbol field => $"{prefix}{field.Name}", IEventSymbol evt => $"{prefix}{evt.Name}", @@ -259,7 +269,11 @@ internal static string FormatParameters(IEnumerable parameters { return string.Join(", ", parameters.Select(p => { - var type = p.Type.ToDisplayString(SymbolDisplayFormat.MinimallyQualifiedFormat); + // Fully qualified: MinimallyQualifiedFormat displayed both A.Foo + // and B.Foo as `Foo`, collapsing distinct overloads onto one key. + var type = p.Type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + if (type.StartsWith("global::", StringComparison.Ordinal)) + type = type["global::".Length..]; return p.RefKind switch { RefKind.Ref => $"ref {type}", @@ -270,6 +284,28 @@ internal static string FormatParameters(IEnumerable parameters })); } + /// + /// Type name with generic arity (Roslyn/ECMA backtick convention): + /// `Foo`, `Foo`1`. Without the arity, `class Foo<T>` and `class Foo` + /// collapse onto one key. + /// + internal static string FormatTypeRef(INamedTypeSymbol t) => + t.Arity > 0 ? $"{t.Name}`{t.Arity}" : t.Name; + + /// + /// Containing-type chain outermost-first including + /// itself, e.g. `Outer`1.Inner`. Only the immediate type name would let + /// `Ns.Outer1.Inner` and `Ns.Outer2.Inner` collide. + /// + internal static string ContainingTypePath(INamedTypeSymbol t) + { + var segments = new List(); + for (var current = (INamedTypeSymbol?)t; current is not null; current = current.ContainingType) + segments.Add(FormatTypeRef(current)); + segments.Reverse(); + return string.Join(".", segments); + } + internal static List LocationToRange(Location loc) { var lineSpan = loc.GetLineSpan(); diff --git a/helpers/csharp/scip-csharp.csproj b/helpers/csharp/scip-csharp.csproj index 0bd5ce91..1dcfb8fa 100644 --- a/helpers/csharp/scip-csharp.csproj +++ b/helpers/csharp/scip-csharp.csproj @@ -47,4 +47,9 @@ + + + + + diff --git a/helpers/csharp/tests/FindRefsWireContractTests.cs b/helpers/csharp/tests/FindRefsWireContractTests.cs new file mode 100644 index 00000000..5be6d8aa --- /dev/null +++ b/helpers/csharp/tests/FindRefsWireContractTests.cs @@ -0,0 +1,70 @@ +using System.Text.Json; +using Xunit; +using ScipCsharp; + +namespace ScipCsharp.Tests; + +/// +/// Pins the JSON wire contract of against the +/// production serializer (): the Warnings list must +/// serialize under the snake_case warnings property, because that is +/// the key the Rust side parses +/// (scip_parse::parse_find_refs_output, #[serde(default)] warnings). +/// A rename or a changed naming policy would silently deserialize as +/// "complete" on the Rust side — a partial answer passing for a full one. +/// +/// The AddRange sites that populate the list +/// (ReferenceResolver.BuildSymbolMapAsync / ResolveReferencesAsync) are +/// Roslyn-path glue covered by the csharp_helper_integration suite +/// (cargo test --features csharp_helper_integration); this test pins the +/// wire shape only. +/// +public class FindRefsWireContractTests +{ + [Fact] + public async Task FindRefsOutput_Warnings_SerializeUnderTheSnakeCaseWarningsProperty() + { + var output = new FindRefsOutput + { + Symbol = "csharp MyApp . Calculator#Add(int, int).", + References = + [ + new FindRefsOccurrence { File = "src/Calc.cs", StartLine = 7, EndLine = 7, Kind = "reference" }, + ], + Warnings = + [ + "could not compile project 'Broken' — its symbols are missing from the map", + ], + }; + + var path = Path.Combine(Path.GetTempPath(), $"findrefs-wire-{Guid.NewGuid():N}.json"); + try + { + // The exact serializer the find-refs subcommand ships with — not + // hand-built options, which would not catch a change to + // OutputWriter's own JsonSerializerOptions. + await OutputWriter.WriteRefsAsync(output, path); + var json = await File.ReadAllTextAsync(path); + var parsed = JsonDocument.Parse(json); + + Assert.True( + parsed.RootElement.TryGetProperty("warnings", out var warnings), + $"the Warnings list must serialize as 'warnings' (SnakeCaseLower), got: {json}"); + Assert.Equal(1, warnings.GetArrayLength()); + Assert.Equal( + "could not compile project 'Broken' — its symbols are missing from the map", + warnings[0].GetString()); + + // The rest of the contract parse_find_refs_output reads. + Assert.Equal("2.0", parsed.RootElement.GetProperty("version").GetString()); + var occurrence = parsed.RootElement.GetProperty("references")[0]; + Assert.True( + occurrence.TryGetProperty("start_line", out _), + "occurrence line fields must stay snake_case"); + } + finally + { + File.Delete(path); + } + } +} diff --git a/helpers/csharp/tests/Fixtures/SmallSolution/Library/Calculator.cs b/helpers/csharp/tests/Fixtures/SmallSolution/Library/Calculator.cs index f07f9454..22be244a 100644 --- a/helpers/csharp/tests/Fixtures/SmallSolution/Library/Calculator.cs +++ b/helpers/csharp/tests/Fixtures/SmallSolution/Library/Calculator.cs @@ -10,6 +10,11 @@ public int Add(int a, int b) return a + b; } + public int Add(int a, int b, int c) + { + return a + b + c; + } + public int Subtract(int a, int b) { return a - b; diff --git a/helpers/csharp/tests/IndexerTests.cs b/helpers/csharp/tests/IndexerTests.cs index 72482237..fa4e93bb 100644 --- a/helpers/csharp/tests/IndexerTests.cs +++ b/helpers/csharp/tests/IndexerTests.cs @@ -25,7 +25,7 @@ public void ScipIndex_SerializesWithSnakeCase() { var index = new ScipIndex { - Metadata = new ScipMetadata { Version = "1.0", ToolInfo = "scip-csharp" }, + Metadata = new ScipMetadata { Version = "2.0", ToolInfo = "scip-csharp" }, Documents = [ new ScipDocument diff --git a/helpers/csharp/tests/KeyFormatTests.cs b/helpers/csharp/tests/KeyFormatTests.cs new file mode 100644 index 00000000..9d4e1bfb --- /dev/null +++ b/helpers/csharp/tests/KeyFormatTests.cs @@ -0,0 +1,164 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using ScipCsharp; +using Xunit; + +namespace ScipCsharp.Tests; + +/// +/// Identity-collision fixtures for +/// (todo #139 B4). Each test compiles a small in-memory snippet with REAL +/// Roslyn symbols — no MSBuild, no workspace, no BCL references; the fixture +/// code declares only its own types — and pins the three key-format +/// guarantees: generic arity, full containing-type paths and fully qualified +/// parameter types keep distinct declarations on distinct keys. +/// +public class KeyFormatTests +{ + /// + /// Compiles and maps every declared named type + /// and ordinary method through the production key builder. + /// + private static Dictionary BuildKeys(string source) + { + var tree = CSharpSyntaxTree.ParseText(source); + var compilation = CSharpCompilation.Create( + "keyformat-fixtures", + [tree], + references: [], + options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + var keys = new Dictionary(SymbolEqualityComparer.Default); + Collect(compilation.SourceModule.GlobalNamespace, keys); + Assert.NotEmpty(keys); + return keys; + } + + private static void Collect(INamespaceSymbol ns, Dictionary keys) + { + foreach (var member in ns.GetMembers()) + { + if (member is INamespaceSymbol child) + Collect(child, keys); + else if (member is INamedTypeSymbol type) + CollectType(type, keys); + } + } + + private static void CollectType(INamedTypeSymbol type, Dictionary keys) + { + var key = SymbolIndexer.SymbolToScipName(type); + if (!string.IsNullOrEmpty(key)) + keys[type] = key; + + foreach (var member in type.GetMembers()) + { + if (member is INamedTypeSymbol nested) + { + CollectType(nested, keys); + } + else if (member is IMethodSymbol { MethodKind: MethodKind.Ordinary } method) + { + var methodKey = SymbolIndexer.SymbolToScipName(method); + if (!string.IsNullOrEmpty(methodKey)) + keys[method] = methodKey; + } + } + } + + [Fact] + public void GenericArityDistinguishesFooFromFooOfT() + { + var keys = BuildKeys(""" + namespace Ns; + class Foo { } + class Foo { } + """); + + var plain = keys.Values.Single(k => k == "csharp Ns . Foo#"); + var generic = keys.Values.Single(k => k == "csharp Ns . Foo`1#"); + Assert.NotEqual(plain, generic); + Assert.Contains("`1", generic); + } + + [Fact] + public void GenericArityDistinguishesMethodOverloads() + { + var keys = BuildKeys(""" + namespace Ns; + class C + { + void M() { } + void M() { } + } + """); + + var plain = keys.Values.Single(k => k == "csharp Ns . C#M()."); + var generic = keys.Values.Single(k => k == "csharp Ns . C#M`1()."); + Assert.NotEqual(plain, generic); + // Arity sits between the name and the parameter list. + Assert.Contains("#M`1(", generic); + } + + [Fact] + public void NestedTypesCarryTheirFullContainingPath() + { + var keys = BuildKeys(""" + namespace Ns; + class Outer1 { class Inner { void M() { } } } + class Outer2 { class Inner { void M() { } } } + """); + + var o1 = keys.Values.Single(k => k == "csharp Ns . Outer1.Inner#M()."); + var o2 = keys.Values.Single(k => k == "csharp Ns . Outer2.Inner#M()."); + Assert.NotEqual(o1, o2); + } + + [Fact] + public void ParameterTypesAreFullyQualified() + { + var keys = BuildKeys(""" + namespace A { class P { } } + namespace B { class P { } } + namespace Ns + { + class C + { + void M(A.P x) { } + void M(B.P y) { } + } + } + """); + + var fromA = keys.Values.Single(k => k == "csharp Ns . C#M(A.P)."); + var fromB = keys.Values.Single(k => k == "csharp Ns . C#M(B.P)."); + Assert.NotEqual(fromA, fromB); + // No bare-param collapse left behind by the qualification. + Assert.DoesNotContain("(P ", fromA); + Assert.DoesNotContain(", P)", fromA); + Assert.DoesNotContain("(P ", fromB); + Assert.DoesNotContain(", P)", fromB); + } + + [Fact] + public void RefKindsAreStillDistinguished() + { + var keys = BuildKeys(""" + namespace A { class P { } } + namespace Ns + { + class C + { + void M(A.P x) { } + void M(ref A.P x) { } + } + } + """); + + // Regression guard: the RefKind switch must survive the switch to the + // fully qualified parameter format. + Assert.Equal( + ["csharp Ns . C#M(A.P).", "csharp Ns . C#M(ref A.P)."], + keys.Values.Where(k => k.Contains("#M(")).OrderBy(k => k, StringComparer.Ordinal).ToList()); + } +} diff --git a/helpers/csharp/tests/ScipCsharp.Tests.csproj b/helpers/csharp/tests/ScipCsharp.Tests.csproj index e83ed9e4..96712ece 100644 --- a/helpers/csharp/tests/ScipCsharp.Tests.csproj +++ b/helpers/csharp/tests/ScipCsharp.Tests.csproj @@ -12,12 +12,22 @@ + + + - + + diff --git a/integrations/claude-code/README.md b/integrations/claude-code/README.md index 859bb657..1831f031 100644 --- a/integrations/claude-code/README.md +++ b/integrations/claude-code/README.md @@ -26,19 +26,38 @@ parent's `AGENTS.md` or the MCP `initialize` instructions at all. ## The fix -Two [Claude Code hooks](https://docs.claude.com/en/docs/claude-code/hooks) +Four [Claude Code hooks](https://docs.claude.com/en/docs/claude-code/hooks) that make the preference *structural* instead of advisory: - **`grep-guard`** — a `PreToolUse` hook on `Grep`. Blocks every `Grep` - call against an internal repo path *for as long as the codesearch serve hub - is reachable*, with a message telling the model exactly how to load and call - codesearch instead. Grep is auto-allowed **only** when codesearch is - genuinely down: the hook probes the unauthenticated `/healthz` liveness - endpoint and lets Grep through only when that probe fails. A low-confidence - or empty codesearch *result* is a successful call ("reformulate"), not a dead - server, so it does **not** unblock Grep. Grep against paths outside the - current repo is never blocked; codesearch doesn't cover arbitrary external - paths well, grep is right there. + call against an indexed repo *for as long as the codesearch serve hub + is reachable*, with a message telling the model exactly how to load and + call codesearch instead. The guarded repo is resolved from the **grep + target itself** — the git root of the path being searched, not the + hook's working directory — so an absolute-path Grep into a different + indexed repo is guarded too (a cwd-based check used to let those + through, and until #199 the absolute-path test itself only recognized + Windows-style roots, so POSIX absolute paths were still resolved + against the cwd). Coverage is decided by **registration**: the target's + git root must be one of the repos in the hub's `~/.codesearch/repos.json` + (honoring `CODESEARCH_REPOS_CONFIG`), which also carves out nested + repos for free — an unregistered clone inside a registered repo + resolves to its own git root and is treated as uncovered. Grep is + auto-allowed **only** when codesearch is genuinely + down: the hook probes the unauthenticated `/healthz` liveness endpoint + and lets Grep through only when that probe fails. A low-confidence or + empty codesearch *result* is a successful call ("reformulate"), not a + dead server, so it does **not** unblock Grep. One exception: when the + target repo is live but **mid-reindex** (the serve watcher's full + refresh, e.g. right after a branch switch), the hook denies Grep with a + **wait-and-retry** instruction (sleep 15-30s, then re-run the + codesearch call) — searching a mid-rebuild index returns stale/empty + results and must not degrade into a manual grep approval on every + routine checkout. The freshness probe (`GET /indexing?path=`) is skipped silently on serves that predate the endpoint, so + hook and server versions can be mixed freely. Grep against paths + outside any git repo, or against repos codesearch does not cover, is + never blocked; grep is right there in those cases. - **`subagent-preamble`** — a `PreToolUse` hook on `Agent` (the subagent-spawn tool). Prepends a short preamble to every subagent prompt explaining that @@ -46,9 +65,32 @@ that make the preference *structural* instead of advisory: and when to prefer it over Grep/Glob. This is the only way to reach subagents at all, since they don't inherit `AGENTS.md` or MCP instructions. -Both hooks fail open: if they can't parse their input, or codesearch isn't -running/indexed, they get out of the way and let Grep proceed untouched. They -never block anything outside the current repo. +- **`edit-guard`** — a `PreToolUse` hook on `Edit`/`Write`/`MultiEdit`. + Blocks an edit to a file in a codesearch-registered repo until codesearch + was consulted for that exact path within the last 5 minutes: + `mcp__codesearch__find_impact` for SCIP-backed languages + (`.cs .ts .tsx .mts .cts`), `mcp__codesearch__find(kind="usages")` for + everything else — making the caller-aware-editing protocol structural. + Coverage uses the same registration model as grep-guard (shared + `codesearch-common` helpers): the file's git root must be listed in + `~/.codesearch/repos.json` (or `CODESEARCH_SERVER` is set); unregistered + repos and non-git paths are never blocked. + +- **`edit-guard-post`** — a `PostToolUse` hook on + `mcp__codesearch__find_impact` and `mcp__codesearch__find`, recording the + "consulted" markers `edit-guard` reads into a state file + (`$TMPDIR/.codesearch-edit-guard-state.json` on bash, + `%TEMP%\.codesearch-edit-guard-state.json` on PowerShell). It fires on + every `find_impact` call and every `find(kind="usages")` (the kind check + is done script-side — matchers only see tool names). Any outcome counts: + "no results" and "no SCIP backend" still mark the path as consulted, so + the guard can never wedge permanently. PostToolUse hooks cannot block + anything; this one never emits a decision and always exits 0. + +All hooks fail open: if they can't parse their input, or codesearch isn't +running/indexed, they get out of the way and let the tool call proceed +untouched. They never block targets outside any git repo, or repos +codesearch does not cover. ## Install @@ -67,9 +109,11 @@ bash integrations/claude-code/install.sh --project ``` The installer: -1. copies the hook scripts into `/hooks/codesearch/` -2. merges two `PreToolUse` registrations into `/settings.json` - (backing up the existing file first) +1. copies the guard scripts, the PostToolUse companion and the shared + `codesearch-common` helpers into `/hooks/codesearch/` +2. merges the `PreToolUse` registrations (one per guard) and the + `PostToolUse` registration (the companion) into + `/settings.json` (backing up the existing file first) 3. is idempotent — re-running it skips hooks already registered and never duplicates or clobbers unrelated settings @@ -77,9 +121,9 @@ Restart Claude Code (or start a new session) after installing. ## Manual install -If you'd rather wire it up by hand, or already have a `PreToolUse.Grep` / -`PreToolUse.Agent` hook and want to merge manually, add to -`~/.claude/settings.json` (or `.claude/settings.json` for project scope): +If you'd rather wire it up by hand, or already have hooks on these events and +want to merge manually, add to `~/.claude/settings.json` (or +`.claude/settings.json` for project scope): ```json { @@ -96,6 +140,20 @@ If you'd rather wire it up by hand, or already have a `PreToolUse.Grep` / "hooks": [ { "type": "command", "command": "pwsh -NoProfile -NonInteractive -File \"/subagent-preamble.ps1\"" } ] + }, + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { "type": "command", "command": "pwsh -NoProfile -NonInteractive -File \"/edit-guard.ps1\"" } + ] + } + ], + "PostToolUse": [ + { + "matcher": "mcp__codesearch__find_impact|mcp__codesearch__find", + "hooks": [ + { "type": "command", "command": "pwsh -NoProfile -NonInteractive -File \"/edit-guard-post.ps1\"" } + ] } ] } @@ -104,32 +162,72 @@ If you'd rather wire it up by hand, or already have a `PreToolUse.Grep` / Use the `.sh` scripts with a `bash "/..."` command instead on macOS/Linux. Point `` at wherever you copy `hooks/*.ps1` / `hooks/*.sh`. +The guard scripts source `codesearch-common.ps1` / `codesearch-common.sh` +from their own directory, so that file must be copied alongside them. ## Uninstall -Remove the two `PreToolUse` entries (matcher `Grep` and `Agent` whose command -points at `hooks/codesearch/`) from `settings.json`, and delete +Remove the guard entries (matchers `Grep`, `Agent`, `WebSearch|WebFetch` and +`Edit|Write|MultiEdit` under `PreToolUse`, and +`mcp__codesearch__find_impact|mcp__codesearch__find` under `PostToolUse`, +whose commands point at `hooks/codesearch/`) from `settings.json`, and delete `/hooks/codesearch/`. ## Caveats -- `grep-guard` detects "codesearch is available **for the current repo**" via - a local `.codesearch.db` at the git root, or an explicit `CODESEARCH_SERVER` - env var for pure remote-serve setups with no local index. It deliberately - does **not** treat "a `codesearch` process is running" as sufficient — - `codesearch serve` commonly runs as a persistent background hub covering - many registered repos (`codesearch index list`), so that process is alive - on a dev machine almost all the time regardless of whether the current - directory is one of the repos it actually indexes. Checking process - presence alone made the hook fire in every directory on the machine, - including unindexed ones — this was found and fixed after exactly that - false-positive showed up in real use. +- `grep-guard` resolves the guarded repo from the **grep target**, never + from its own cwd: empty/relative paths resolve against the cwd repo + (they are relative to it by definition), absolute paths resolve against + the git root of the path being searched. Coverage is then decided by + **registration with the serve hub**: the target's git root must be one + of the repos listed in `~/.codesearch/repos.json` (the same + registration list the hub itself resolves queries by, honoring the + `CODESEARCH_REPOS_CONFIG` override), or an explicit `CODESEARCH_SERVER` + env var for pure remote-serve setups with no local registration. A + local `.codesearch.db` directory is deliberately **not** a coverage + signal anymore (#199): a stale db from a since-unregistered repo used + to deny Grep even though the hub could not answer for that repo + (unknown alias), and a registered repo whose db directory was gone + slipped through uncovered. Because the git *root* must equal a + registration, nested repos are carved out correctly: an unregistered + clone inside a registered repo resolves to its own root and is treated + as uncovered. Missing, unreadable or malformed `repos.json` (or a + missing `jq`) fails **open** — a guard that cannot resolve coverage + must allow, never deny. It deliberately does **not** treat "a + `codesearch` process is running" as sufficient — `codesearch serve` + commonly runs as a persistent background hub covering many registered + repos (`codesearch index list`), so that process is alive on a dev + machine almost all the time regardless of whether the searched repo is + one of the repos it actually indexes. Checking process presence alone + made the hook fire in every directory on the machine, including + unindexed ones — this was found and fixed after exactly that + false-positive showed up in real use. (The older cwd-based resolution + had the mirror-image defect: an absolute-path Grep into a *different* + indexed repo looked "external" and slipped the guard — also fixed, + same release. Until #199 the absolute-path test itself only matched + Windows-style roots — `C:\`, `C:/`, MSYS `/c/`, UNC `//server` — so + POSIX absolute paths like `/home/...` were still resolved against the + cwd; with a session cwd that is a plain parent directory the guard + silently allowed everything.) If your setup connects to a remote `codesearch serve` instance with no - local `.codesearch.db`, set `CODESEARCH_SERVER` to opt back into - enforcement for that repo. + local `repos.json` registration, set `CODESEARCH_SERVER` to opt back + into enforcement for that repo. Both the bash and the PowerShell twin + now resolve coverage through the shared `codesearch-common` helpers, + so the two shells behave identically (the PowerShell twin used to lag + behind on the `.codesearch.db`/Windows-only-path signals — closed with + the edit-guard work, #199). +- `edit-guard` is per-file, not per-repo: a marker for one file never + unblocks an edit to another file. The marker state lives in a temp-dir + JSON file (`$TMPDIR/.codesearch-edit-guard-state.json` / + `%TEMP%\.codesearch-edit-guard-state.json`), entries expire after + 5 minutes and are pruned on write; missing or corrupt state counts as + "not consulted" (deny on covered repos, allow everywhere else). Like + the other guards it is per-machine, not per-repo: it only ever fires + on files whose git root is registered with the serve hub. - Both hooks are per-machine, not per-repo: install once at user scope and - every project benefits, including ones without a local `.codesearch.db` - (the guard simply won't block Grep there, since step 2 fails open). + every project benefits, including ones not registered with the serve + hub (the guard simply won't block Grep there, since coverage fails + open). - `grep-guard` decides "is codesearch down?" by probing the serve hub's unauthenticated `/healthz` endpoint (base URL from `CODESEARCH_SERVER`, else `http://127.0.0.1:$CODESEARCH_SERVE_PORT`, else the compiled default diff --git a/integrations/claude-code/hooks/codesearch-common.ps1 b/integrations/claude-code/hooks/codesearch-common.ps1 new file mode 100644 index 00000000..00c01d89 --- /dev/null +++ b/integrations/claude-code/hooks/codesearch-common.ps1 @@ -0,0 +1,90 @@ +# Shared helpers for the codesearch Claude Code guard hooks. Dot-sourced by +# grep-guard.ps1, edit-guard.ps1 and edit-guard-post.ps1: +# . (Join-Path $PSScriptRoot 'codesearch-common.ps1') +# +# PowerShell twin of codesearch-common.sh — keep the two behaviorally +# equivalent. Coverage model: a repo is covered when its git root is +# REGISTERED in ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides +# the location), or when CODESEARCH_SERVER opts a pure remote-serve setup +# in. Everything here fails open: an unresolvable coverage question must +# allow, never deny. + +function Get-CodesearchReposConfigFile { + if ($env:CODESEARCH_REPOS_CONFIG) { return $env:CODESEARCH_REPOS_CONFIG } + return (Join-Path $HOME '.codesearch/repos.json') +} + +function ConvertTo-NormRepoPath { + param([string]$P) + $p = $P + if ($p.StartsWith('\\?\')) { $p = $p.Substring(4) } + $p = $p -replace '\\', '/' + while ($p -ne '/' -and $p.EndsWith('/')) { $p = $p.TrimEnd('/') } + return $p +} + +function Test-CodesearchRepoPathEqual { + param([string]$A, [string]$B) + $a = ConvertTo-NormRepoPath $A + $b = ConvertTo-NormRepoPath $B + # Exact on POSIX-style paths; case-insensitive for Windows drive-letter + # paths (NTFS folds case throughout), mirroring codesearch-common.sh. + if ($a -match '^[A-Za-z]:/') { $a = $a.ToLowerInvariant() } + if ($b -match '^[A-Za-z]:/') { $b = $b.ToLowerInvariant() } + return [string]::Equals($a, $b, [System.StringComparison]::Ordinal) +} + +function Test-CodesearchTargetRegistered { + param([string]$Root) + if ($env:CODESEARCH_SERVER) { return $true } + $cfg = Get-CodesearchReposConfigFile + if ([string]::IsNullOrWhiteSpace($cfg)) { return $false } + if (-not (Test-Path -LiteralPath $cfg -PathType Leaf)) { return $false } + try { $json = Get-Content -LiteralPath $cfg -Raw | ConvertFrom-Json } catch { return $false } + if ($null -eq $json -or $null -eq $json.repos) { return $false } + foreach ($prop in $json.repos.PSObject.Properties) { + $val = $prop.Value + $reg = if ($val -is [string]) { $val } else { $val | ConvertTo-Json -Compress -Depth 10 } + if ([string]::IsNullOrEmpty($reg)) { continue } + if (Test-CodesearchRepoPathEqual $reg $Root) { return $true } + } + return $false +} + +function Resolve-TargetGitRoot { + param([string]$Path) + $root = $null + $norm = $Path.TrimEnd('/', '\') + if ($norm -match '^([A-Za-z]:[\\/]|/)') { + # Absolute path (Windows drive, UNC, or any POSIX root): the git root + # OF THE TARGET, not of the hook's cwd. + $probe = $norm + try { + if (Test-Path -LiteralPath $probe -PathType Leaf) { $probe = Split-Path -Parent $probe } + } catch {} + try { + $gr = (& git -C $probe rev-parse --show-toplevel 2>$null) + if ($LASTEXITCODE -eq 0 -and $gr) { $root = "$gr".Trim() } + } catch {} + } else { + # Empty or relative path: resolves against the cwd repo. + try { + $gr = (& git rev-parse --show-toplevel 2>$null) + if ($LASTEXITCODE -eq 0 -and $gr) { $root = "$gr".Trim() } + } catch {} + } + return $root +} + +function ConvertTo-StateKey { + param([string]$P) + if ([string]::IsNullOrEmpty($P)) { return '' } + $p = $P + $norm = $p.TrimEnd('/', '\') + if (-not ($norm -match '^([A-Za-z]:[\\/]|/)')) { + $p = (Join-Path (Get-Location).Path $p) + } + $k = ConvertTo-NormRepoPath $p + if ($k -match '^[A-Za-z]:/') { $k = $k.ToLowerInvariant() } + return $k +} diff --git a/integrations/claude-code/hooks/codesearch-common.sh b/integrations/claude-code/hooks/codesearch-common.sh new file mode 100644 index 00000000..a77e51ac --- /dev/null +++ b/integrations/claude-code/hooks/codesearch-common.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Shared helpers for the codesearch Claude Code guard hooks. Sourced (never +# executed) by grep-guard.sh, edit-guard.sh and edit-guard-post.sh: +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/codesearch-common.sh" +# +# Coverage model (shared with the Rust serve hub, src/db_discovery/repos.rs): +# a repo is covered when its git root is REGISTERED in +# ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides the location), +# or when CODESEARCH_SERVER opts a pure remote-serve setup in. Everything +# here fails open: an unresolvable coverage question must allow, never deny. +# Requires: jq + +# jq.exe builds on Windows emit CRLF; command substitution strips only the +# \n, leaving a stray \r that breaks every following string comparison. +# Every jq -r read pipes through this. +jq_str() { + tr -d '\r' +} + +# repos.json location — mirrors src/db_discovery/repos.rs `config_path()`: +# CODESEARCH_REPOS_CONFIG override > ~/.codesearch/repos.json. +repos_config_file() { + if [ -n "${CODESEARCH_REPOS_CONFIG:-}" ]; then + printf '%s' "$CODESEARCH_REPOS_CONFIG" + else + printf '%s' "${HOME:-}/.codesearch/repos.json" + fi +} + +# Normalize one path for comparison: drop a Windows extended-length prefix +# (\\?\ — exactly 4 chars), unify backslashes to forward slashes (Git-Bash +# reports C:/x/y while repos.json records "C:\\x\\y"), and trim trailing +# separators. Registration canonicalizes paths before writing them +# (safe_canonicalize), so after this both sides agree byte-for-byte on +# POSIX and component-wise on Windows. +norm_repo_path() { + local p="$1" + p="${p%$'\r'}" + case "$p" in + '\\?\'*) p="${p:4}" ;; + esac + p="${p//\\//}" + while [ "$p" != "/" ] && [ "${p%/}" != "$p" ]; do + p="${p%/}" + done + printf '%s' "$p" +} + +# Path equality: exact on POSIX, case-insensitive for Windows drive-letter +# paths (NTFS is case-insensitive throughout) — mirrors the serve hub's own +# /indexing resolver, which folds case on Windows only. +repo_path_eq() { + local a b + a="$(norm_repo_path "$1")" + b="$(norm_repo_path "$2")" + case "$a" in [A-Za-z]:/*) a="$(printf '%s' "$a" | tr '[:upper:]' '[:lower:]')" ;; esac + case "$b" in [A-Za-z]:/*) b="$(printf '%s' "$b" | tr '[:upper:]' '[:lower:]')" ;; esac + [ "$a" = "$b" ] +} + +# Is this git root covered by codesearch — registered with the local serve +# hub (repos.json), or a CODESEARCH_SERVER opt-in for pure remote-serve +# setups? Fails OPEN on any resolver problem (missing/unreadable/malformed +# repos.json, missing jq): a guard that cannot resolve coverage must allow, +# never deny. +target_registered() { + local root="$1" cfg reg + [ -n "${CODESEARCH_SERVER:-}" ] && return 0 + cfg="$(repos_config_file)" + [ -n "$cfg" ] || return 1 + [ -r "$cfg" ] || return 1 + while IFS= read -r reg; do + [ -n "$reg" ] || continue + if repo_path_eq "$reg" "$root"; then + return 0 + fi + # jq gets the config via stdin redirection, never as a positional path + # argument — native jq.exe builds mangle POSIX-style paths (see the + # same note in web-guard.sh). + done < <(jq -r '(.repos // {}) | to_entries[] | .value | tostring' < "$cfg" 2>/dev/null) + return 1 +} + +# Git root of the repo a target path lives in — the TARGET's repo, never the +# hook's cwd one, for absolute paths; empty/relative paths are relative to +# the cwd by definition and resolve against it. Prints the root; empty when +# the path is outside any git repo (or git is unusable). +resolve_target_git_root() { + local p="$1" norm probe root="" + norm="${p%/}" + norm="${norm%\\}" + case "$norm" in + # `/*` matches every absolute path — Windows drive, UNC and POSIX + # alike (a Windows-style-only pattern used to strand POSIX absolute + # paths on the cwd branch, #199). + [A-Za-z]:[\\/]*|/*) + probe="$norm" + [ -f "$probe" ] && probe="$(dirname "$probe")" + root=$(git -C "$probe" rev-parse --show-toplevel 2>/dev/null || true) + ;; + *) + root=$(git rev-parse --show-toplevel 2>/dev/null || true) + ;; + esac + printf '%s' "$root" +} + +# Canonical state-file key for a file path: edit-guard reads the state by +# this key and edit-guard-post writes it, so both sides must derive the SAME +# key from whatever path form the agent happened to pass. Absolute-izes +# relative paths against the cwd, then reuses the repos.json normalization +# plus repo_path_eq's Windows drive-letter casefold. +norm_state_key() { + local p="$1" n + [ -n "$p" ] || return 0 + case "$p" in + [A-Za-z]:[\\/]*|/*) ;; + *) p="$PWD/${p#./}" ;; + esac + n="$(norm_repo_path "$p")" + case "$n" in [A-Za-z]:/*) n="$(printf '%s' "$n" | tr '[:upper:]' '[:lower:]')" ;; esac + printf '%s' "$n" +} diff --git a/integrations/claude-code/hooks/edit-guard-post.ps1 b/integrations/claude-code/hooks/edit-guard-post.ps1 new file mode 100644 index 00000000..d1022ce5 --- /dev/null +++ b/integrations/claude-code/hooks/edit-guard-post.ps1 @@ -0,0 +1,74 @@ +# PostToolUse companion for edit-guard: records a "codesearch consulted" +# marker per file path after qualifying MCP calls, into the same state file +# edit-guard reads. Windows twin of edit-guard-post.sh — keep the two +# behaviorally equivalent. Fires on: +# - mcp__codesearch__find_impact (ANY outcome counts — including "no +# results" / "no SCIP backend": counting failures too is what keeps the +# guard from blocking permanently) +# - mcp__codesearch__find with kind="usages" (string compare HERE — the +# matcher regex can only filter on tool name) +# +# PostToolUse hooks cannot block anything: no decision is emitted and this +# script ALWAYS exits 0. Symbol-only find_impact calls carry no file-ish +# input field and are skipped (nothing to attribute a path to). + +$ErrorActionPreference = 'Stop' + +try { + . (Join-Path $PSScriptRoot 'codesearch-common.ps1') + $raw = [Console]::In.ReadToEnd() + if ([string]::IsNullOrWhiteSpace($raw)) { exit 0 } + $data = $raw | ConvertFrom-Json +} catch { + exit 0 +} + +$tool = $data.tool_name +$inp = $data.tool_input + +switch ($tool) { + 'mcp__codesearch__find_impact' { $markTool = 'find_impact' } + 'mcp__codesearch__find' { + $kind = if ($inp -and @($inp.PSObject.Properties.Name) -contains 'kind') { [string]$inp.kind } else { '' } + if ($kind -ne 'usages') { exit 0 } + $markTool = 'find_usages' + } + default { exit 0 } +} + +$p = $null +foreach ($field in @('file', 'path', 'file_path')) { + if ($inp -and @($inp.PSObject.Properties.Name) -contains $field) { + $candidate = [string]$inp.$field + if (-not [string]::IsNullOrEmpty($candidate)) { $p = $candidate; break } + } +} +if ([string]::IsNullOrEmpty($p)) { exit 0 } + +$key = ConvertTo-StateKey $p +if ([string]::IsNullOrEmpty($key)) { exit 0 } + +$stateFile = Join-Path $env:TEMP '.codesearch-edit-guard-state.json' +$window = 300 +$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + +# Upsert, pruning expired entries; corrupt/unreadable state starts over. +$state = @{} +if (Test-Path -LiteralPath $stateFile -PathType Leaf) { + try { + $stored = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json + foreach ($pr in $stored.PSObject.Properties) { + $ts = 0L + try { $ts = [long]$pr.Value.ts } catch {} + if (($now - $ts) -lt $window) { + $state[$pr.Name] = @{ tool = [string]$pr.Value.tool; ts = $ts } + } + } + } catch {} +} +$state[$key] = @{ tool = $markTool; ts = $now } +try { + $state | ConvertTo-Json -Depth 5 -Compress | Set-Content -LiteralPath $stateFile -NoNewline +} catch {} + +exit 0 diff --git a/integrations/claude-code/hooks/edit-guard-post.sh b/integrations/claude-code/hooks/edit-guard-post.sh new file mode 100644 index 00000000..381267c6 --- /dev/null +++ b/integrations/claude-code/hooks/edit-guard-post.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# PostToolUse companion for edit-guard: records a "codesearch consulted" +# marker per file path after qualifying MCP calls, into the same state file +# edit-guard reads. Fires on: +# - mcp__codesearch__find_impact (ANY outcome counts — including +# "no results" / "no SCIP backend": the guard cannot know the result, and +# counting failures too is what keeps it from blocking permanently) +# - mcp__codesearch__find with kind="usages" +# (the kind check is a string compare HERE, not in the matcher regex — +# matchers filter on tool name only) +# +# PostToolUse hooks cannot block anything: this script never emits a +# decision and ALWAYS exits 0. Symbol-only find_impact calls carry no +# file-ish input field and are skipped (nothing to attribute a path to). +# Requires: jq + +set -euo pipefail + +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/codesearch-common.sh" + +raw="$(cat)" +[ -z "$raw" ] && exit 0 + +tool=$(echo "$raw" | jq -r '.tool_name // empty' 2>/dev/null | jq_str) +case "$tool" in + mcp__codesearch__find_impact) + mark_tool="find_impact" + ;; + mcp__codesearch__find) + kind=$(echo "$raw" | jq -r '.tool_input.kind // empty' 2>/dev/null | jq_str) + [ "$kind" = "usages" ] || exit 0 + mark_tool="find_usages" + ;; + *) exit 0 ;; +esac + +# find_impact names its target via `file`/`path`; find uses `path`. Accept +# any of them — FIRST NON-EMPTY (jq's `//` treats "" as truthy, so the +# empties are filtered explicitly; the ps1 twin does the same) — so both +# tools attribute to the same key. +p=$(echo "$raw" | jq -r ' + [ .tool_input.file, .tool_input.path, .tool_input.file_path ] + | map(select(. != null and . != "")) + | .[0] // empty +' 2>/dev/null | jq_str) +[ -z "$p" ] && exit 0 + +key="$(norm_state_key "$p")" +[ -z "$key" ] && exit 0 + +state_file="${TMPDIR:-/tmp}/.codesearch-edit-guard-state.json" +window=300 +now=$(date +%s) + +# Upsert, pruning expired entries so the file cannot grow without bound; a +# window-expired entry is simply dropped and the fresh write re-creates it. +tmp="$(mktemp)" +merged=false +if [ -f "$state_file" ]; then + if jq --arg k "$key" --arg t "$mark_tool" --argjson now "$now" --argjson window "$window" ' + with_entries( + (.value.ts // -1 | (tonumber? // -1)) as $ts + | select(($now - $ts) < $window) + ) + { ($k): { tool: $t, ts: $now } } + ' < "$state_file" > "$tmp" 2>/dev/null && [ -s "$tmp" ]; then + merged=true + fi +fi +if [ "$merged" != true ]; then + # Missing or corrupt state: start over from this single marker. + jq -n --arg k "$key" --arg t "$mark_tool" --argjson now "$now" \ + '{ ($k): { tool: $t, ts: $now } }' > "$tmp" 2>/dev/null || true +fi +mv -f "$tmp" "$state_file" 2>/dev/null || true +rm -f "$tmp" 2>/dev/null || true + +exit 0 diff --git a/integrations/claude-code/hooks/edit-guard.ps1 b/integrations/claude-code/hooks/edit-guard.ps1 new file mode 100644 index 00000000..a45b0af6 --- /dev/null +++ b/integrations/claude-code/hooks/edit-guard.ps1 @@ -0,0 +1,118 @@ +# PreToolUse hook: require a recent codesearch consultation before edits. +# Fires on Edit/Write/MultiEdit. Windows twin of edit-guard.sh — keep the +# two behaviorally equivalent. +# +# When the edited file's repo is codesearch-registered (git root listed in +# ~/.codesearch/repos.json, honoring CODESEARCH_REPOS_CONFIG, or a +# CODESEARCH_SERVER opt-in), every touched file needs a marker proving the +# agent consulted codesearch for exactly that path within the last 5 +# minutes: mcp__codesearch__find_impact for SCIP-backed languages +# (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for +# everything else. Markers are written by the edit-guard-post PostToolUse +# companion into $env:TEMP\.codesearch-edit-guard-state.json. +# +# Lenient acceptance: ANY marker for the path within the window lets the +# edit through — the marker proves the agent consulted codesearch for this +# file, which is the point. +# +# Fail-open on: no target paths, paths outside any git repo, unregistered +# repos, or a crashed hook — all allow. Missing/corrupt state counts as NOT +# consulted (deny on covered repos, allow everywhere else); so does an +# expired marker, and the next consultation refreshes it. +# +# Install: see ../README.md (or run `codesearch hooks claude install`). + +$ErrorActionPreference = 'Stop' + +try { + . (Join-Path $PSScriptRoot 'codesearch-common.ps1') + $raw = [Console]::In.ReadToEnd() + if ([string]::IsNullOrWhiteSpace($raw)) { exit 0 } + $data = $raw | ConvertFrom-Json +} catch { + exit 0 # never block a tool call because the hook failed to parse its own input +} + +$tool = $data.tool_name +$inp = $data.tool_input + +if ($tool -ne 'Edit' -and $tool -ne 'Write' -and $tool -ne 'MultiEdit') { exit 0 } +if ($null -eq $inp) { exit 0 } + +# Target paths: the primary file_path, plus (defensively) per-edit entries. +$candidates = @() +$names = @($inp.PSObject.Properties.Name) +if ($names -contains 'file_path') { $candidates += [string]$inp.file_path } +if (($names -contains 'edits') -and $null -ne $inp.edits) { + foreach ($e in @($inp.edits)) { + if ($null -ne $e -and @($e.PSObject.Properties.Name) -contains 'file_path') { + $candidates += [string]$e.file_path + } + } +} +$paths = @($candidates | Where-Object { -not [string]::IsNullOrEmpty($_) } | Select-Object -Unique) +if ($paths.Count -eq 0) { exit 0 } # nothing attributable -> allow (fail-open) + +$stateFile = Join-Path $env:TEMP '.codesearch-edit-guard-state.json' +$window = 300 +$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + +function Test-CheckedRecently { + param([string]$Key) + if (-not (Test-Path -LiteralPath $stateFile -PathType Leaf)) { return $false } + try { + $state = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json + $entry = $state.PSObject.Properties[$Key] + if ($null -eq $entry) { return $false } + $ts = [long]$entry.Value.ts + return (($now - $ts) -lt $window) + } catch { + return $false # corrupt/unreadable state counts as absent + } +} + +$failPath = $null +$failTool = $null +foreach ($p in $paths) { + $root = Resolve-TargetGitRoot $p + if (-not $root) { continue } # outside any git repo -> allow + if (-not (Test-CodesearchTargetRegistered $root)) { continue } # unregistered -> fail-open + $key = ConvertTo-StateKey $p + if (Test-CheckedRecently $key) { continue } + $failPath = $p + $ext = [System.IO.Path]::GetExtension($p).TrimStart('.').ToLowerInvariant() + if (@('cs', 'ts', 'tsx', 'mts', 'cts') -contains $ext) { + $failTool = 'mcp__codesearch__find_impact (SCIP-backed language)' + } else { + $failTool = 'mcp__codesearch__find(symbol, kind="usages")' + } + break +} + +if ($null -eq $failPath) { exit 0 } # every path checked recently -> allow + +$msg = @" +edit-guard: this edit needs a codesearch consultation first. + +Blocked path: $failPath +Required call: $failTool — on that exact file. + +Any outcome counts: "no results" and "no SCIP backend" still prove you +consulted codesearch for this file. Run the call, then retry the SAME +edit — it stays allowed for 5 minutes. + +Why: find_impact (C#/TS) / find kind="usages" (other languages) before +edits keeps refactors caller-aware; this guard makes that protocol +structural for codesearch-registered repos. Unregistered repos, non-git +paths and unparseable events fail open and are never blocked. +"@ + +$out = @{ + hookSpecificOutput = @{ + hookEventName = 'PreToolUse' + permissionDecision = 'deny' + permissionDecisionReason = $msg + } +} +$out | ConvertTo-Json -Depth 10 -Compress +exit 0 diff --git a/integrations/claude-code/hooks/edit-guard.sh b/integrations/claude-code/hooks/edit-guard.sh new file mode 100644 index 00000000..c37da6f5 --- /dev/null +++ b/integrations/claude-code/hooks/edit-guard.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# PreToolUse hook: require a recent codesearch consultation before edits. +# Fires on Edit/Write/MultiEdit. Bash twin of edit-guard.ps1. +# Requires: jq +# +# When the edited file's repo is codesearch-registered (shared coverage model +# with grep-guard: git root listed in ~/.codesearch/repos.json, honoring +# CODESEARCH_REPOS_CONFIG, or a CODESEARCH_SERVER opt-in), every touched file +# needs a marker proving the agent consulted codesearch for exactly that path +# within the last 5 minutes: mcp__codesearch__find_impact for SCIP-backed +# languages (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for +# everything else. The markers are written by the edit-guard-post PostToolUse +# companion into ${TMPDIR:-/tmp}/.codesearch-edit-guard-state.json. +# +# Lenient acceptance: ANY marker for the path within the window lets the edit +# through, regardless of which tool wrote it — the marker proves the agent +# consulted codesearch for this file, which is the point; policing +# find_impact-vs-find_usages per extension would deny edits after a +# legitimate-but-"wrong-kind" lookup. +# +# Fail-open on: no target paths, paths outside any git repo, unregistered +# repos, or a crashed hook — all allow. Missing/corrupt state counts as NOT +# consulted (deny on covered repos, allow everywhere else); so does an +# expired marker, and the next consultation refreshes it (no state +# pollution). +# +# Install: see ../README.md (or run `codesearch hooks claude install`). + +set -euo pipefail + +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/codesearch-common.sh" + +raw="$(cat)" +[ -z "$raw" ] && exit 0 + +tool=$(echo "$raw" | jq -r '.tool_name // empty' 2>/dev/null | jq_str) +case "$tool" in + Edit | Write | MultiEdit) ;; + *) exit 0 ;; +esac + +# Target paths: the primary file_path, plus (defensively) any per-edit +# file_path entries a MultiEdit-style payload may carry. +paths=$(echo "$raw" | jq -r ' + [ .tool_input.file_path, + (.tool_input.edits[]?.file_path // empty) + ] + | map(select(. != null and . != "")) + | unique | .[] +' 2>/dev/null | jq_str) +[ -z "$paths" ] && exit 0 # nothing attributable -> allow (fail-open) + +state_file="${TMPDIR:-/tmp}/.codesearch-edit-guard-state.json" +window=300 +now=$(date +%s) + +# The marker only proves "codesearch was consulted recently for this path"; +# a non-numeric ts is corruption, not a stale check. +checked_recently() { + local key="$1" entry ts + [ -f "$state_file" ] || return 1 + entry=$(jq -rc --arg k "$key" '.[$k] // empty' < "$state_file" 2>/dev/null | jq_str) + [ -n "$entry" ] || return 1 + ts=$(printf '%s' "$entry" | jq -r '.ts // empty' 2>/dev/null | jq_str) + case "$ts" in '' | *[!0-9]*) return 1 ;; esac + [ $((now - ts)) -lt "$window" ] +} + +fail_path="" +fail_tool="" +while IFS= read -r p; do + [ -n "$p" ] || continue + root="$(resolve_target_git_root "$p")" + [ -z "$root" ] && continue # outside any git repo -> allow + target_registered "$root" || continue # unregistered repo -> allow (fail-open) + key="$(norm_state_key "$p")" + if ! checked_recently "$key"; then + fail_path="$p" + ext=$(printf '%s' "${p##*.}" | tr '[:upper:]' '[:lower:]') + case "$ext" in + cs | ts | tsx | mts | cts) fail_tool='mcp__codesearch__find_impact (SCIP-backed language)' ;; + *) fail_tool='mcp__codesearch__find(symbol, kind="usages")' ;; + esac + break + fi +done <<< "$paths" + +[ -z "$fail_path" ] && exit 0 # every path checked recently -> allow + +msg=$(cat < fail open, never block. +try { + . (Join-Path $PSScriptRoot 'codesearch-common.ps1') +} catch { + exit 0 +} + try { $raw = [Console]::In.ReadToEnd() if ([string]::IsNullOrWhiteSpace($raw)) { exit 0 } @@ -52,62 +69,25 @@ $names = @($inp.PSObject.Properties.Name) $path = if ($names -contains 'path') { [string]$inp.path } else { '' } # ------------------------------------------------------------------ -# 1. Is the path internal to the current repo? -# ------------------------------------------------------------------ -$isInternal = $true -if ($path -and $path -ne '.' -and $path -ne './') { - $normPath = $path.TrimEnd('/\') - # Absolute paths (Windows drive letter, or Git-Bash /c/... style) - if ($normPath -match '^([A-Za-z]:[\\/]|/[a-zA-Z]/|//)') { - try { - $gr = (& git rev-parse --show-toplevel 2>$null) - if ($LASTEXITCODE -eq 0 -and $gr) { - $gr = $gr.Trim() -replace '[/\\]', [System.IO.Path]::DirectorySeparatorChar - $abs = $normPath -replace '[/\\]', [System.IO.Path]::DirectorySeparatorChar - if (-not $abs.StartsWith($gr, [System.StringComparison]::OrdinalIgnoreCase)) { - $isInternal = $false - } - } - } catch { - $isInternal = $false # can't determine git root -> assume external, allow grep - } - } - # Relative paths ("src/", "../sibling/") stay internal = $true -} - -if (-not $isInternal) { exit 0 } - -# ------------------------------------------------------------------ -# 2. Does codesearch COVER this repo? Don't block if it doesn't. +# 1+2. Resolve the TARGET repo (the repo this Grep is aimed at) and check +# codesearch coverage THERE — never in the hook's cwd. # -# NOTE: we deliberately do NOT treat "a codesearch process is running" as -# sufficient. codesearch commonly runs as a persistent background `serve` -# hub covering many registered repos (`codesearch index list`) — that -# process is alive nearly all the time on a dev machine, regardless of -# whether the CURRENT directory is one of the repos it actually indexes. -# Using process-presence alone made this hook fire in every directory on -# the machine, including ones with no index at all. A local `.codesearch.db` -# at the git root is the precise, fast signal that THIS repo is indexed. +# The resolution (absolute path -> the target's own git root, empty/ +# relative -> the cwd repo) lives in codesearch-common.ps1 and is shared +# with edit-guard. History (#54, #199): see codesearch-common.sh. # ------------------------------------------------------------------ -function Test-CodesearchCoversRepo { - try { - $gr = (& git rev-parse --show-toplevel 2>$null) - if ($LASTEXITCODE -eq 0 -and $gr) { - $gr = $gr.Trim() - if (Test-Path (Join-Path $gr '.codesearch.db')) { return $true } - } - } catch {} - - # Explicit opt-in escape hatch for pure remote-serve setups with no local - # .codesearch.db (this repo's index lives only on a remote `codesearch - # serve` host). Requires the user to consciously set this env var, so it - # can't spuriously fire the way "any process running" did. - if ($env:CODESEARCH_SERVER) { return $true } +$targetRoot = Resolve-TargetGitRoot $path - return $false -} +# Not inside any git repo (or git unusable) -> external target: grep is right. +if (-not $targetRoot) { exit 0 } -if (-not (Test-CodesearchCoversRepo)) { exit 0 } +# Coverage: the TARGET repo's git root is REGISTERED with the serve hub +# (~/.codesearch/repos.json — the same list the hub itself resolves by), +# matching the bash twin and edit-guard; a running serve hub alone is NOT a +# signal (it covers many repos and being alive says nothing about this one). +# CODESEARCH_SERVER stays the explicit opt-in for pure remote-serve setups. +# Fails open: missing/malformed repos.json counts as unregistered -> allow. +if (-not (Test-CodesearchTargetRegistered $targetRoot)) { exit 0 } # ------------------------------------------------------------------ # 3. Is the codesearch serve hub actually UP right now? (Liveness probe.) @@ -133,10 +113,10 @@ function Get-CodesearchBaseUrl { } function Test-CodesearchLive { - $base = Get-CodesearchBaseUrl + param([string]$Base) try { # Short timeout keeps grep latency low; /healthz answers instantly. - $null = Invoke-WebRequest -Uri "$base/healthz" -TimeoutSec 2 -UseBasicParsing + $null = Invoke-WebRequest -Uri "$Base/healthz" -TimeoutSec 2 -UseBasicParsing return $true } catch { # An HTTP error RESPONSE (4xx/5xx) still proves the server is reachable @@ -149,7 +129,58 @@ function Test-CodesearchLive { } # codesearch is DOWN -> grep is genuinely all you have, let it through. -if (-not (Test-CodesearchLive)) { exit 0 } +$base = Get-CodesearchBaseUrl +if (-not (Test-CodesearchLive -Base $base)) { exit 0 } + +# ------------------------------------------------------------------ +# 3.5 Is the TARGET repo mid-reindex right now? (Freshness probe, #55.) +# +# Liveness (/healthz) says the server is UP; it says nothing about index +# FRESHNESS. Right after a branch switch the serve watcher fires a full +# refresh, and searches against the mid-rebuild index return stale/empty +# results — which used to push the agent into a manual grep approval on +# every routine checkout (the exact "stale after branch switch" report). +# GET /indexing?path= resolves the target to its registered repo +# and reports an active reindex. When one is in flight, deny with a +# WAIT-AND-RETRY instruction: the tree did not change, the index is just +# catching up — waiting beats grepping. +# +# Backward compat: an older serve without this endpoint answers 404, the +# probe is skipped (catch below), and behaviour is exactly the pre-#55 deny. +# ------------------------------------------------------------------ +try { + $enc = [uri]::EscapeDataString(($targetRoot -replace '\\', '/')) + $resp = Invoke-WebRequest -Uri "$base/indexing?path=$enc" -TimeoutSec 2 -UseBasicParsing + $fresh = $resp.Content | ConvertFrom-Json + if ($fresh.covered -eq $true -and $fresh.indexing -eq $true) { + $waitMsg = @" +codesearch is LIVE, but THIS repo's index is being rebuilt right now (a +branch switch or file change fired the serve watcher's full refresh). +Searching immediately would return stale or empty results — that is the +rebuild in progress, not a miss, and NOT a reason to grep. + +WAIT 15-30 seconds (Bash: sleep 20), then RETRY your codesearch call — it +will answer normally once the rebuild lands. The working tree did not +change; only the index is catching up, so grep adds nothing here. + +If the rebuild still has not landed after ~2 minutes, run your codesearch +call anyway (a partially fresh index still beats grep) or ask the user how +to proceed. +"@ + $waitOut = @{ + hookSpecificOutput = @{ + hookEventName = 'PreToolUse' + permissionDecision = 'deny' + permissionDecisionReason = $waitMsg + } + } + $waitOut | ConvertTo-Json -Depth 10 -Compress + exit 0 + } +} catch { + # 404 (older serve) or probe failure: no freshness signal — fall through + # to the standard deny below. +} # ------------------------------------------------------------------ # 4. Block with actionable guidance @@ -186,7 +217,8 @@ error, you MUST pass project="" (single repo) or group="" available_groups — pick from that list (the alias may differ from the folder name). -Grep is always allowed for paths OUTSIDE the current repo. +Grep is always allowed for targets outside any git repo, and for repos +codesearch does not cover (no local index, no CODESEARCH_SERVER). "@ $out = @{ diff --git a/integrations/claude-code/hooks/grep-guard.sh b/integrations/claude-code/hooks/grep-guard.sh index ad27f3cd..cd24b3b1 100644 --- a/integrations/claude-code/hooks/grep-guard.sh +++ b/integrations/claude-code/hooks/grep-guard.sh @@ -11,65 +11,76 @@ # and leaked grep on every low-confidence result. We now probe the # unauthenticated /healthz liveness endpoint directly. # +# The target repo is resolved from the GREP TARGET itself (its own git root), +# never from the hook's cwd — absolute paths into a different repo resolve +# against THAT repo (#54, and see the #199 note below: POSIX absolute paths +# used to slip through this very test). Coverage is decided by REGISTRATION +# with the local serve hub (repos.json), not by a .codesearch.db directory +# (#199). When the target repo is covered and LIVE but MID-REINDEX +# (branch-switch full refresh, #55), the deny is a WAIT-AND-RETRY +# instruction instead: searching a mid-rebuild index returns stale/empty +# results and must not degrade into a manual grep approval on every checkout. +# # Install: see ../README.md (or run ../install.sh to wire this up automatically). set -euo pipefail +# Shared coverage/target-resolution helpers (repos.json registration, path +# normalization) live in codesearch-common.sh, shared with edit-guard. +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/codesearch-common.sh" + raw="$(cat)" [ -z "$raw" ] && exit 0 -tool=$(echo "$raw" | jq -r '.tool_name // empty') +tool=$(echo "$raw" | jq -r '.tool_name // empty' | jq_str) [ "$tool" != "Grep" ] && exit 0 -path=$(echo "$raw" | jq -r '.tool_input.path // empty') - -# ------------------------------------------------------------------ -# 1. Is the path internal to the current repo? -# ------------------------------------------------------------------ -is_internal=true -if [ -n "$path" ] && [ "$path" != "." ] && [ "$path" != "./" ]; then - case "$path" in - /*) - git_root=$(git rev-parse --show-toplevel 2>/dev/null || true) - if [ -n "$git_root" ]; then - case "$path" in - "$git_root"*) is_internal=true ;; - *) is_internal=false ;; - esac - else - is_internal=false # can't determine git root -> assume external, allow grep - fi - ;; - *) is_internal=true ;; # relative path stays internal - esac -fi - -[ "$is_internal" = false ] && exit 0 +path=$(echo "$raw" | jq -r '.tool_input.path // empty' | jq_str) # ------------------------------------------------------------------ -# 2. Does codesearch COVER this repo? Don't block if it doesn't. +# 1+2. Resolve the TARGET repo (the repo this Grep is aimed at) and check +# codesearch coverage THERE — never in the hook's cwd. +# +# History (#54): coverage used to be resolved from the hook's cwd. An +# absolute-path Grep into a DIFFERENT indexed repo then failed the +# startswith(cwd-repo-root) test, looked "external", and was allowed even +# though its target repo was fully covered. The guard now follows the +# target: empty/relative paths resolve against the cwd repo (they are +# relative to it by definition), absolute paths resolve against the git +# root of the path itself. # -# NOTE: we deliberately do NOT treat "a codesearch process is running" as -# sufficient. codesearch commonly runs as a persistent background `serve` -# hub covering many registered repos (`codesearch index list`) — that -# process is alive nearly all the time on a dev machine, regardless of -# whether the CURRENT directory is one of the repos it actually indexes. -# Using process-presence alone made this hook fire in every directory on -# the machine, including ones with no index at all. A local `.codesearch.db` -# at the git root is the precise, fast signal that THIS repo is indexed. +# History (#199): the #54 rewrite still mis-binned POSIX absolute paths +# (/home/..., /tmp/...): its absolute-detection pattern only matched +# Windows-style roots (C:\, C:/, MSYS /c/, UNC //server), so a POSIX +# absolute target fell into the relative branch and resolved against the +# hook's cwd — with a session cwd that is not itself a git repo (a parent +# directory holding several repos) the target resolved to NOTHING, looked +# external, and the guard silently allowed every grep. `/*` now covers +# every absolute path. +# +# Coverage signal (#199): the target's git root is REGISTERED with the +# local serve hub (~/.codesearch/repos.json — the same registration list +# the hub itself resolves queries by). A `.codesearch.db` directory at +# the git root was only ever a proxy for that and is wrong in both +# directions: a stale db from a since-unregistered repo denied Grep while +# the hub could not actually answer for it (unknown alias), and a +# registered repo whose db directory was gone slipped through uncovered. +# Matching the git ROOT (never a path prefix) also carves out nested +# repos for free: an unregistered clone nested inside a registered repo +# resolves to its OWN git root, equals no registration, and is correctly +# treated as uncovered. The explicit CODESEARCH_SERVER opt-in stays for +# pure remote-serve setups with no local repos.json (#199 tracks that it +# is a URL override rather than a coverage signal — to be reworked +# separately). # ------------------------------------------------------------------ -codesearch_covers=false -git_root=$(git rev-parse --show-toplevel 2>/dev/null || true) -if [ -n "$git_root" ] && [ -d "$git_root/.codesearch.db" ]; then - codesearch_covers=true -elif [ -n "${CODESEARCH_SERVER:-}" ]; then - # Explicit opt-in escape hatch for pure remote-serve setups with no local - # .codesearch.db. Requires the user to consciously set this env var, so - # it can't spuriously fire the way "any process running" did. - codesearch_covers=true -fi +target_root="$(resolve_target_git_root "$path")" -[ "$codesearch_covers" = false ] && exit 0 +# Not inside any git repo (or git unusable) -> external target: grep is right. +[ -z "$target_root" ] && exit 0 + +# Covered = the target's git root is registered with the serve hub +# (repos.json), or CODESEARCH_SERVER opts a pure remote-serve setup in. +target_registered "$target_root" || exit 0 # ------------------------------------------------------------------ # 3. Is the codesearch serve hub actually UP right now? (Liveness probe.) @@ -105,6 +116,55 @@ if ! curl -sS --max-time 2 "${base}/healthz" >/dev/null 2>&1; then exit 0 # codesearch is DOWN -> grep is genuinely all you have, let it through fi +# ------------------------------------------------------------------ +# 3.5 Is the TARGET repo mid-reindex right now? (Freshness probe, #55.) +# +# Liveness (/healthz) says the server is UP; it says nothing about index +# FRESHNESS. Right after a branch switch the serve watcher fires a full +# refresh, and searches against the mid-rebuild index return stale/empty +# results — which used to push the agent into a manual grep approval on +# every routine checkout. GET /indexing?path= resolves the +# target to its registered repo and reports an active reindex. When one is +# in flight, deny with a WAIT-AND-RETRY instruction: the tree did not +# change, the index is just catching up — waiting beats grepping. +# +# Backward compat: an older serve without this endpoint answers 404; the +# probe is skipped and behaviour is exactly the pre-#55 deny. The ?path= +# value is percent-encoded (jq @uri) — a repo root containing spaces, +, +# &, # or non-ASCII would otherwise make the probe fail or answer for the +# WRONG repo prefix, silently disabling the wait-and-retry path. +# ------------------------------------------------------------------ +enc=$(printf '%s' "$target_root" | jq -sRr @uri) +if fresh_json=$(curl -sS --max-time 2 "${base}/indexing?path=${enc}" 2>/dev/null); then + covered=$(echo "$fresh_json" | jq -r '.covered // false' 2>/dev/null || echo false) + indexing=$(echo "$fresh_json" | jq -r '.indexing // false' 2>/dev/null || echo false) + if [ "$covered" = "true" ] && [ "$indexing" = "true" ]; then + msg=$(cat <<'EOF' +codesearch is LIVE, but THIS repo's index is being rebuilt right now (a +branch switch or file change fired the serve watcher's full refresh). +Searching immediately would return stale or empty results — that is the +rebuild in progress, not a miss, and NOT a reason to grep. + +WAIT 15-30 seconds (Bash: sleep 20), then RETRY your codesearch call — it +will answer normally once the rebuild lands. The working tree did not +change; only the index is catching up, so grep adds nothing here. + +If the rebuild still has not landed after ~2 minutes, run your codesearch +call anyway (a partially fresh index still beats grep) or ask the user how +to proceed. +EOF +) + jq -n --arg msg "$msg" '{ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + permissionDecisionReason: $msg + } + }' + exit 0 + fi +fi + # ------------------------------------------------------------------ # 4. Block with actionable guidance # ------------------------------------------------------------------ @@ -140,7 +200,9 @@ error, you MUST pass project="" (single repo) or group="" available_groups — pick from that list (the alias may differ from the folder name). -Grep is always allowed for paths OUTSIDE the current repo. +Grep is always allowed for targets outside any git repo, for repos that are +not registered with the codesearch serve hub (repos.json), and for pure +remote-serve setups opted in via CODESEARCH_SERVER. EOF ) diff --git a/integrations/claude-code/hooks/run-tests.sh b/integrations/claude-code/hooks/run-tests.sh new file mode 100644 index 00000000..e536bdc3 --- /dev/null +++ b/integrations/claude-code/hooks/run-tests.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Self-test suite for the codesearch Claude Code guard hooks: edit-guard + +# its edit-guard-post PostToolUse companion, plus a grep-guard smoke case +# pinning the codesearch-common.sh extraction. NOT wired into cargo or CI — +# run manually: +# bash integrations/claude-code/hooks/run-tests.sh +# Requires: bash, jq, git (and curl for the grep-guard smoke case). +# +# Coverage is simulated without a real index: repoA is "registered" via a +# temp repos.json passed through CODESEARCH_REPOS_CONFIG (the same override +# the guards honour at runtime), repoB is not listed. The serve hub is only +# contacted by the grep-guard smoke case, which forces a dead port so the +# answer is deterministic. State lives in a temp TMPDIR; the env overrides +# are process-scoped and die with this script. + +set -u + +HOOKS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$HOOKS_DIR/codesearch-common.sh" + +PASS=0 +FAIL=0 +TMP_ROOT="$(mktemp -d)" +STATE_DIR="$TMP_ROOT/state" +mkdir -p "$STATE_DIR" + +cleanup() { + unset CODESEARCH_REPOS_CONFIG CODESEARCH_SERVER TMPDIR + rm -rf "$TMP_ROOT" +} +trap cleanup EXIT + +REPO_A="$TMP_ROOT/repoA" +REPO_B="$TMP_ROOT/repoB" +mkdir -p "$REPO_A" "$REPO_B" +git init -q "$REPO_A" +git init -q "$REPO_B" +touch "$REPO_A/x.cs" "$REPO_A/m.ts" "$REPO_A/p.py" "$REPO_B/b.cs" +ROOT_A="$(git -C "$REPO_A" rev-parse --show-toplevel)" +ROOT_B="$(git -C "$REPO_B" rev-parse --show-toplevel)" + +REPOS_JSON="$TMP_ROOT/repos.json" +jq -n --arg a "$ROOT_A" '{repos: {repoA: $a}}' > "$REPOS_JSON" + +export TMPDIR="$STATE_DIR" +export CODESEARCH_REPOS_CONFIG="$REPOS_JSON" +unset CODESEARCH_SERVER + +ok() { PASS=$((PASS + 1)); echo "PASS: $1"; } +bad() { FAIL=$((FAIL + 1)); echo "FAIL: $1"; } + +run_edit() { OUT="$(printf '%s' "$1" | bash "$HOOKS_DIR/edit-guard.sh" 2>/dev/null)"; } +run_post() { printf '%s' "$1" | bash "$HOOKS_DIR/edit-guard-post.sh" >/dev/null 2>&1; } + +edit_event() { jq -n --arg p "$1" '{tool_name: "Edit", tool_input: {file_path: $p}}'; } +multiedit_event() { + jq -n --arg a "$1" --arg b "$2" \ + '{tool_name: "MultiEdit", tool_input: {file_path: $a, edits: [{file_path: $a}, {file_path: $b}]}}' +} +post_find_impact() { jq -n --arg f "$1" '{tool_name: "mcp__codesearch__find_impact", tool_input: {file: $f}}'; } +post_find() { jq -n --arg f "$1" --arg k "$2" '{tool_name: "mcp__codesearch__find", tool_input: {path: $f, kind: $k}}'; } + +expect_allow() { + if [ -z "$OUT" ]; then + ok "$1" + else + bad "$1 — expected silent allow, got: $OUT" + fi +} +expect_deny() { # $1 = label, $2 = optional substring the reason must contain + local d="" reason="" + if [ -n "$OUT" ]; then + d="$(printf '%s' "$OUT" | jq -r '.hookSpecificOutput.permissionDecision // empty' 2>/dev/null | jq_str 2>/dev/null)" + reason="$(printf '%s' "$OUT" | jq -r '.hookSpecificOutput.permissionDecisionReason // empty' 2>/dev/null | jq_str 2>/dev/null)" + fi + if [ "$d" != "deny" ]; then + bad "$1 — expected deny, got: ${OUT:-}" + return + fi + if [ $# -ge 2 ] && ! printf '%s' "$reason" | grep -q "$2"; then + bad "$1 — deny message missing '$2'" + return + fi + ok "$1" +} + +case_no_coverage_allows() { + run_edit "$(edit_event "$REPO_B/b.cs")" + expect_allow "unregistered repo: Edit .cs fails open (silent allow)" +} +case_covered_cs_denied() { + run_edit "$(edit_event "$REPO_A/x.cs")" + expect_deny "covered repo: Edit .cs denied (find_impact required)" "find_impact" +} +case_covered_ts_denied() { + run_edit "$(edit_event "$REPO_A/m.ts")" + expect_deny "covered repo: Edit .ts denied (find_impact required)" "find_impact" +} +case_covered_py_needs_usages() { + run_edit "$(edit_event "$REPO_A/p.py")" + expect_deny "covered repo: Edit .py denied (find kind=usages required)" 'kind="usages"' +} +case_definition_does_not_mark() { + run_post "$(post_find "$REPO_A/m.ts" "definition")" + run_edit "$(edit_event "$REPO_A/m.ts")" + expect_deny "find kind=definition does not mark: Edit .ts still denied" "find_impact" +} +case_find_impact_marker_allows() { + run_post "$(post_find_impact "$REPO_A/x.cs")" + run_edit "$(edit_event "$REPO_A/x.cs")" + expect_allow "after find_impact marker: Edit .cs allowed" +} +case_empty_first_field_falls_through() { + # jq's `//` treats "" as truthy: the post hook must filter empties + # explicitly or {"file":"","path":...} writes NO marker while the ps1 + # twin writes one (shell drift, review round 1). + run_post "$(jq -n --arg p "$REPO_A/m.ts" \ + '{tool_name: "mcp__codesearch__find_impact", tool_input: {file: "", path: $p}}')" + run_edit "$(edit_event "$REPO_A/m.ts")" + expect_allow "empty-string file field falls through to path: marker written" +} +case_multiedit_partial_marks_denied() { + run_edit "$(multiedit_event "$REPO_A/x.cs" "$REPO_A/p.py")" + expect_deny "MultiEdit: marked .cs + unmarked .py denied, failing path named" "$REPO_A/p.py" +} +case_find_usages_marker_allows() { + run_post "$(post_find "$REPO_A/p.py" "usages")" + run_edit "$(edit_event "$REPO_A/p.py")" + expect_allow "after find(kind=usages) marker: Edit .py allowed" + run_edit "$(multiedit_event "$REPO_A/x.cs" "$REPO_A/p.py")" + expect_allow "MultiEdit with every path marked allowed" +} +case_window_expiry_denies_again() { + local sf="$STATE_DIR/.codesearch-edit-guard-state.json" + local old=$(( $(date +%s) - 400 )) + jq --argjson old "$old" 'with_entries(.value.ts = $old)' < "$sf" > "$sf.new" && + mv "$sf.new" "$sf" + run_edit "$(edit_event "$REPO_A/x.cs")" + expect_deny "expired marker (>5 min): Edit .cs denied again" "find_impact" +} +case_grep_guard_smoke() { + # Regression pin for the codesearch-common.sh extraction: grep-guard + # still resolves + guards the covered repo, and still auto-allows when + # the hub is unreachable (dead port -> /healthz probe fails). + local out + out="$(printf '{"tool_name":"Grep","tool_input":{"path":"%s"}}' "$REPO_A" | + CODESEARCH_SERVE_PORT=1 bash "$HOOKS_DIR/grep-guard.sh" 2>/dev/null)" + if [ -z "$out" ]; then + ok "grep-guard smoke: covered repo + hub down -> allow" + else + bad "grep-guard smoke — expected silent allow, got: $out" + fi +} + +case_no_coverage_allows +case_covered_cs_denied +case_covered_ts_denied +case_covered_py_needs_usages +case_definition_does_not_mark +case_find_impact_marker_allows +case_empty_first_field_falls_through +case_multiedit_partial_marks_denied +case_find_usages_marker_allows +case_window_expiry_denies_again +case_grep_guard_smoke + +echo +echo "edit-guard self-tests: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/integrations/claude-code/hooks/subagent-preamble.ps1 b/integrations/claude-code/hooks/subagent-preamble.ps1 index dea1a957..19509018 100644 --- a/integrations/claude-code/hooks/subagent-preamble.ps1 +++ b/integrations/claude-code/hooks/subagent-preamble.ps1 @@ -64,6 +64,11 @@ add project="" (single repo) or group="" (cross-repo). The error response lists the valid available_projects / available_groups — pick from that list; the alias may differ from the folder name. +EDIT RULE: before editing code in a codesearch-registered repo, consult it +for the file first — mcp__codesearch__find_impact (C#/TS) or +mcp__codesearch__find(kind="usages") (other languages). The edit-guard hook +blocks Edit/Write/MultiEdit on that file until you do (5-minute window). + Fall back to Grep/Glob only after codesearch returns no useful results, or when the path is outside the current repo (codesearch covers internal paths only unless you're in multi-repo serve mode with an explicit group). diff --git a/integrations/claude-code/hooks/subagent-preamble.sh b/integrations/claude-code/hooks/subagent-preamble.sh index f1845b93..f4364b53 100644 --- a/integrations/claude-code/hooks/subagent-preamble.sh +++ b/integrations/claude-code/hooks/subagent-preamble.sh @@ -46,6 +46,11 @@ add project="" (single repo) or group="" (cross-repo). The error response lists the valid available_projects / available_groups — pick from that list; the alias may differ from the folder name. +EDIT RULE: before editing code in a codesearch-registered repo, consult it +for the file first — mcp__codesearch__find_impact (C#/TS) or +mcp__codesearch__find(kind="usages") (other languages). The edit-guard hook +blocks Edit/Write/MultiEdit on that file until you do (5-minute window). + Fall back to Grep/Glob only after codesearch returns no useful results, or when the path is outside the current repo (codesearch covers internal paths only unless you're in multi-repo serve mode with an explicit group). diff --git a/integrations/claude-code/hooks/web-guard.ps1 b/integrations/claude-code/hooks/web-guard.ps1 index db7a7e13..6dddaac1 100644 --- a/integrations/claude-code/hooks/web-guard.ps1 +++ b/integrations/claude-code/hooks/web-guard.ps1 @@ -5,13 +5,41 @@ # API / docs questions more precisely — and more currently — than an open web # search. Nothing structurally stops the model from reaching for the always-on # WebSearch/WebFetch tools first, so this hook makes the preference structural: -# the FIRST WebSearch/WebFetch is blocked with actionable guidance; if the same -# query is retried within 5 minutes (i.e. the mounts didn't have the answer), it -# is let through. +# the FIRST WebSearch/WebFetch *about a mounted product* is blocked with +# actionable guidance; a further call about that same mount within 5 minutes +# (i.e. the mounts didn't have the answer) is let through. +# +# TOPIC SCOPING (the important part): a mount only earns the right to intercept +# queries about ITS OWN subject. Mounting cloud/acme says "I have the Acme +# docs indexed" — it says nothing about Rust, Claude Code, or Azure CLI. So the +# hook first decides whether the query is plausibly about a mounted product, and +# stays out of the way entirely when it isn't. A mount's alias is its keyword +# (cloud/acme -> "acme"); anything else it should answer for is declared in +# repos.json under `.remote_mount_topics` — see README. # # Passes through (exit 0, no block) when: # - there are NO remote mounts to steer toward (nothing indexed to prefer) -# - the same query was already blocked in the last 5 minutes +# - the query/URL doesn't mention any mounted product or its declared topics. +# The mounts are a small opt-in allowlist of specific vendor doc sets, so +# they cannot answer a question about an unrelated vendor, a language +# runtime, or a model card. Blocking such a call buys nothing and costs a +# guaranteed wasted round-trip. +# - the same MOUNT was already steered for in the last 5 minutes. Keyed on the +# matched mount, NOT on the exact query string, on purpose: the natural +# follow-up after "the mirror had nothing" is a REFINED web query, and an +# exact-string key treats every refinement as a fresh first attempt and +# blocks it again — punishing precisely the correct behaviour. +# +# TWO SILENT-DEGRADATION TRAPS, both of which turn this hook into a no-op +# without any visible error (an "allow" is silent, so a broken gate looks +# exactly like a working one): +# - NEVER split a mount name with .Split(@('/','\'), [StringSplitOptions]...): +# PowerShell coerces that array into the single-String separator overload and +# looks for the literal "/ \", so nothing ever splits, no alias is ever +# derived, and the gate matches NOTHING. Use `-split` with a regex, or +# .Substring(.LastIndexOf('/') + 1) as below. +# - ALWAYS strip CR from values read out of repos.json. A trailing \r silently +# survives into the alias ("acme`r"), which then never matches anything. # # Windows twin of web-guard.sh. # @@ -33,11 +61,13 @@ $inp = $data.tool_input if ($tool -ne 'WebSearch' -and $tool -ne 'WebFetch') { exit 0 } if ($null -eq $inp) { exit 0 } -# Query (WebSearch) or target URL (WebFetch) — used for the cache key + guidance. +# Query (WebSearch) or target URL (WebFetch) — used for matching, the cache key +# and the guidance text. $names = @($inp.PSObject.Properties.Name) $q = if ($names -contains 'query') { [string]$inp.query } elseif ($names -contains 'url') { [string]$inp.url } else { '' } +if ([string]::IsNullOrWhiteSpace($q)) { exit 0 } # ------------------------------------------------------------------ # 1. Are there any remote doc mounts to steer toward? @@ -50,28 +80,93 @@ $config = if ($env:CODESEARCH_REPOS_CONFIG) { $env:CODESEARCH_REPOS_CONFIG } if (-not (Test-Path $config)) { exit 0 } $mountList = @() +$topicMap = $null try { $cfg = Get-Content $config -Raw | ConvertFrom-Json if ($cfg.PSObject.Properties.Name -contains 'remote_mounts' -and $cfg.remote_mounts) { - $mountList = @($cfg.remote_mounts) + # Strip CR — see the trap note in the header. + $mountList = @($cfg.remote_mounts | ForEach-Object { ([string]$_).Trim("`r", "`n", ' ') } | + Where-Object { $_ }) + } + if ($cfg.PSObject.Properties.Name -contains 'remote_mount_topics') { + $topicMap = $cfg.remote_mount_topics } } catch { exit 0 # unreadable/invalid config -> don't get in the way } if ($mountList.Count -eq 0) { exit 0 } -$mounts = $mountList -join ', ' # ------------------------------------------------------------------ -# 2. Retry cache: same query blocked recently -> let it through. +# 2. Is this query actually ABOUT one of the mounted products? +# +# Keywords per mount: the alias itself, plus any extra terms declared in +# `.remote_mount_topics` (keyed by the full "/" or the bare alias). +# Matching is case-insensitive on a word-ish boundary, so "acme" hits +# "Acme DAM API" and "help.acme.com" but not "acmexyz". +# +# No match -> this simply isn't the mirror's subject -> allow, silently. +# ------------------------------------------------------------------ +$haystack = $q.ToLowerInvariant() + +function Test-KeywordMatch { + param([string]$Haystack, [string]$Keyword) + if ([string]::IsNullOrWhiteSpace($Keyword)) { return $false } + $pattern = '(^|[^a-z0-9])' + [regex]::Escape($Keyword) + '([^a-z0-9]|$)' + return [regex]::IsMatch($Haystack, $pattern, [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) +} + +function Get-MountTopics { + param($TopicMap, [string]$Mount, [string]$Alias) + if ($null -eq $TopicMap) { return @() } + foreach ($key in @($Mount, $Alias)) { + $prop = $TopicMap.PSObject.Properties[$key] + if ($prop -and $prop.Value) { + return @($prop.Value | ForEach-Object { ([string]$_).Trim("`r", "`n", ' ') } | + Where-Object { $_ }) + } + } + return @() +} + +$matched = @() +foreach ($mount in $mountList) { + # Alias = the part after the last '/' — the PowerShell equivalent of the + # shell twin's ${mount##*/}. See the header for why .Split(@('/','\'), ...) + # must NOT be used here. + $aliasName = ([string]$mount).Substring(([string]$mount).LastIndexOf('/') + 1) + + $hit = $false + if (Test-KeywordMatch -Haystack $haystack -Keyword $aliasName) { + $hit = $true + } else { + foreach ($topic in (Get-MountTopics -TopicMap $topicMap -Mount $mount -Alias $aliasName)) { + if (Test-KeywordMatch -Haystack $haystack -Keyword $topic) { + $hit = $true + break + } + } + } + + if ($hit) { $matched += $mount } +} + +if ($matched.Count -eq 0) { exit 0 } + +$relevant = $matched -join ', ' +$primary = $matched[0] + +# ------------------------------------------------------------------ +# 3. Retry cache: same mount steered for recently -> let it through. +# Covers "tried the mounts, they had nothing, now use the web". # ------------------------------------------------------------------ $cacheFile = Join-Path $env:TEMP '.codesearch-web-guard.json' $cacheTTL = 300 # seconds +$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() $cache = @{} if (Test-Path $cacheFile) { try { $stored = Get-Content $cacheFile -Raw | ConvertFrom-Json - $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() foreach ($prop in $stored.PSObject.Properties) { if (($now - [long]$prop.Value) -lt $cacheTTL) { $cache[$prop.Name] = [long]$prop.Value @@ -80,37 +175,45 @@ if (Test-Path $cacheFile) { } catch {} } -$cacheKey = "$q" +# Keyed on tool + primary matched mount, NOT the raw query: one steer per vendor +# per window. An exact-query key made every refinement of the search terms look +# like a first attempt and blocked it again, which is the opposite of the intent. +$cacheKey = "$tool|$primary" if ($cache.ContainsKey($cacheKey)) { - exit 0 # already blocked once this window -> allow the retry + exit 0 # already steered once this window -> allow the follow-up } -$cache[$cacheKey] = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() +$cache[$cacheKey] = $now try { $cache | ConvertTo-Json -Compress | Set-Content $cacheFile -NoNewline } catch {} # ------------------------------------------------------------------ -# 3. Block with actionable guidance. +# 4. Block with actionable guidance — naming ONLY the mounts that matched. # ------------------------------------------------------------------ $msg = @" -codesearch has remote documentation mounts — search those before the web. -Mounted remotes: $mounts +This looks like a question about a product whose documentation codesearch has +indexed — search that mirror before the web. +Relevant mount(s): $relevant These indexed mounts often answer product/API/docs questions more precisely -(and more currently) than a web search. Try codesearch first. +(and more currently) than a web search, and they cover vendor sites that need a +login and would fail an anonymous fetch anyway. Step 1 — load the deferred MCP tool schemas (one-time per conversation): ToolSearch("select:mcp__codesearch__search,mcp__codesearch__get_chunk") Step 2 — search the relevant mount (compact=false reads matching content inline): - mcp__codesearch__search(query="$q", project="", compact=false) - mcp__codesearch__get_chunk(chunk_ref="") # full context + mcp__codesearch__search(query="$q", project="$primary", compact=false) + mcp__codesearch__get_chunk(chunk_ref="${primary}:") # full context -Pick the relevant project from the mounted remotes above. +For a canonical source link, read the doc's front-matter chunk (start_line 0) +and cite its ``url:`` field verbatim rather than reconstructing a URL. -This exact $tool call is auto-unblocked if you retry it within 5 minutes -(i.e. the mounts didn't have the answer — go ahead and use the web). +If the mount does NOT have the answer, go straight to the web: any further +$tool call about '$primary' is allowed for the next 5 minutes. You do NOT need +to repeat this call verbatim — refining your search terms is fine and will not +be blocked again. "@ $out = @{ diff --git a/integrations/claude-code/hooks/web-guard.sh b/integrations/claude-code/hooks/web-guard.sh index 3db6663f..6c72a633 100644 --- a/integrations/claude-code/hooks/web-guard.sh +++ b/integrations/claude-code/hooks/web-guard.sh @@ -6,13 +6,26 @@ # API / docs questions more precisely — and more currently — than an open web # search. Nothing structurally stops the model from reaching for the always-on # WebSearch/WebFetch tools first, so this hook makes the preference structural: -# the FIRST WebSearch/WebFetch is blocked with actionable guidance; if the same -# query is retried within 5 minutes (i.e. the mounts didn't have the answer), it -# is let through. +# the FIRST WebSearch/WebFetch *about a mounted product* is blocked with +# actionable guidance; a further call about that same mount within 5 minutes +# (i.e. the mounts didn't have the answer) is let through. +# +# TOPIC SCOPING (the important part): a mount only earns the right to intercept +# queries about ITS OWN subject. Mounting cloud/acme says "I have the Acme +# docs indexed" — it says nothing about Rust, Claude Code, or Azure CLI. So the +# hook first decides whether the query is plausibly about a mounted product, and +# stays out of the way entirely when it isn't. A mount's alias is its keyword +# (cloud/acme -> "acme"); anything else it should answer for is declared in +# repos.json under `.remote_mount_topics` — see README. # # Passes through (exit 0, no block) when: # - there are NO remote mounts to steer toward (nothing indexed to prefer) -# - the same query was already blocked in the last 5 minutes +# - the query/URL doesn't mention any mounted product or its declared topics +# - the same MOUNT was already steered for in the last 5 minutes. Keyed on the +# matched mount, NOT on the exact query string, on purpose: the natural +# follow-up after "the mirror had nothing" is a REFINED web query, and an +# exact-string key treats every refinement as a fresh first attempt and +# blocks it again — punishing precisely the correct behaviour. # # Bash/macOS/Linux twin of web-guard.ps1. Requires: jq # @@ -29,8 +42,10 @@ case "$tool" in *) exit 0 ;; esac -# Query (WebSearch) or target URL (WebFetch) — used for the cache key + guidance. +# Query (WebSearch) or target URL (WebFetch) — used for matching, the cache key +# and the guidance text. q=$(echo "$raw" | jq -r '.tool_input.query // .tool_input.url // empty') +[ -z "$q" ] && exit 0 # ------------------------------------------------------------------ # 1. Are there any remote doc mounts to steer toward? @@ -42,11 +57,61 @@ q=$(echo "$raw" | jq -r '.tool_input.query // .tool_input.url // empty') config="${CODESEARCH_REPOS_CONFIG:-$HOME/.codesearch/repos.json}" [ -f "$config" ] || exit 0 -mounts=$(jq -r '(.remote_mounts // []) | join(", ")' < "$config" 2>/dev/null || true) -[ -z "$mounts" ] && exit 0 +# NOTE: strip CR. On Windows/Git Bash jq emits CRLF, and a trailing \r silently +# survives into the alias ("acme\r"), which then never matches anything — +# the hook degrades to "never block" without any visible error. +mapfile -t mount_list < <(jq -r '(.remote_mounts // [])[]' < "$config" 2>/dev/null | tr -d '\r' || true) +[ "${#mount_list[@]}" -eq 0 ] && exit 0 + +# ------------------------------------------------------------------ +# 2. Is this query actually ABOUT one of the mounted products? +# +# Keywords per mount: the alias itself, plus any extra terms declared in +# `.remote_mount_topics` (keyed by the full "/" or the bare alias). +# Matching is case-insensitive on a word-ish boundary, so "acme" hits +# "Acme DAM API" and "help.acme.com" but not "acmexyz". +# +# No match -> this simply isn't the mirror's subject -> allow, silently. +# ------------------------------------------------------------------ +hay=$(printf '%s' "$q" | tr '[:upper:]' '[:lower:]') + +matches_kw() { + local kw="$1" + [ -z "$kw" ] && return 1 + [[ "$hay" =~ (^|[^a-z0-9])"$kw"([^a-z0-9]|$) ]] +} + +matched=() +for mount in "${mount_list[@]}"; do + [ -z "$mount" ] && continue + alias_name="${mount##*/}" + + hit=0 + if matches_kw "$(printf '%s' "$alias_name" | tr '[:upper:]' '[:lower:]')"; then + hit=1 + else + while IFS= read -r topic; do + [ -z "$topic" ] && continue + if matches_kw "$(printf '%s' "$topic" | tr '[:upper:]' '[:lower:]')"; then + hit=1 + break + fi + done < <(jq -r --arg m "$mount" --arg a "$alias_name" \ + '((.remote_mount_topics // {}) | (.[$m] // .[$a] // []))[]' \ + < "$config" 2>/dev/null | tr -d '\r' || true) + fi + + [ "$hit" -eq 1 ] && matched+=("$mount") +done + +[ "${#matched[@]}" -eq 0 ] && exit 0 + +relevant=$(printf '%s, ' "${matched[@]}") +relevant="${relevant%, }" +primary="${matched[0]}" # ------------------------------------------------------------------ -# 2. Retry cache: same query blocked recently -> let it through. +# 3. Retry cache: same mount steered for recently -> let it through. # Covers "tried the mounts, they had nothing, now use the web". # # NOTE: feed the cache file to jq via stdin redirection (`< file`), never as a @@ -55,7 +120,10 @@ mounts=$(jq -r '(.remote_mounts // []) | join(", ")' < "$config" 2>/dev/null || cache_file="${TMPDIR:-/tmp}/.codesearch-web-guard.json" cache_ttl=300 now=$(date +%s) -cache_key="$q" +# Keyed on tool + primary matched mount, NOT the raw query: one steer per vendor +# per window. An exact-query key made every refinement of the search terms look +# like a first attempt and blocked it again, which is the opposite of the intent. +cache_key="$tool|$primary" if [ -f "$cache_file" ]; then blocked_at=$(jq -r --arg k "$cache_key" '.[$k] // empty' < "$cache_file" 2>/dev/null || true) @@ -75,26 +143,31 @@ else fi # ------------------------------------------------------------------ -# 3. Block with actionable guidance. +# 4. Block with actionable guidance — naming ONLY the mounts that matched. # ------------------------------------------------------------------ msg=$(cat < grep-guard.ps1, Agent -> subagent-preamble.ps1) +# 2. Merges the guard registrations into ~/.claude/settings.json: +# PreToolUse: Grep -> grep-guard, Agent -> subagent-preamble, +# WebSearch|WebFetch -> web-guard, Edit|Write|MultiEdit -> edit-guard; +# PostToolUse: find_impact|find -> edit-guard-post # 3. Backs up settings.json before touching it # # Safe to re-run: registrations are matched by command string and skipped if @@ -41,10 +43,15 @@ New-Item -ItemType Directory -Force -Path $hooksDest | Out-Null Copy-Item -Path (Join-Path $hooksSrc 'grep-guard.ps1') -Destination $hooksDest -Force Copy-Item -Path (Join-Path $hooksSrc 'subagent-preamble.ps1') -Destination $hooksDest -Force Copy-Item -Path (Join-Path $hooksSrc 'web-guard.ps1') -Destination $hooksDest -Force +Copy-Item -Path (Join-Path $hooksSrc 'edit-guard.ps1') -Destination $hooksDest -Force +Copy-Item -Path (Join-Path $hooksSrc 'edit-guard-post.ps1') -Destination $hooksDest -Force +Copy-Item -Path (Join-Path $hooksSrc 'codesearch-common.ps1') -Destination $hooksDest -Force -$grepGuardCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/grep-guard.ps1`"" -$preambleCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/subagent-preamble.ps1`"" -$webGuardCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/web-guard.ps1`"" +$grepGuardCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/grep-guard.ps1`"" +$preambleCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/subagent-preamble.ps1`"" +$webGuardCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/web-guard.ps1`"" +$editGuardCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/edit-guard.ps1`"" +$editGuardPostCmd = "pwsh -NoProfile -NonInteractive -File `"$($hooksDest -replace '\\','/')/edit-guard-post.ps1`"" # Load or initialize settings.json if (Test-Path $settingsPath) { @@ -57,30 +64,39 @@ if (Test-Path $settingsPath) { $settings = @{} } -if (-not $settings.ContainsKey('hooks')) { $settings['hooks'] = @{} } +if (-not $settings.ContainsKey('hooks')) { $settings['hooks'] = @{} } if (-not $settings['hooks'].ContainsKey('PreToolUse')) { $settings['hooks']['PreToolUse'] = @() } +if (-not $settings['hooks'].ContainsKey('PostToolUse')) { $settings['hooks']['PostToolUse'] = @() } -$preToolUse = [System.Collections.ArrayList]$settings['hooks']['PreToolUse'] +$preToolUse = [System.Collections.ArrayList]$settings['hooks']['PreToolUse'] +$postToolUse = [System.Collections.ArrayList]$settings['hooks']['PostToolUse'] -function Add-MatcherHook($matcher, $command) { - # Skip if a hook with this exact command already exists anywhere in PreToolUse - foreach ($entry in $preToolUse) { +function Add-MatcherHook { + param([string]$HookEvent, $HookList, [string]$Matcher, [string]$Command) + # Skip if a hook with this exact command already exists anywhere in the list + foreach ($entry in $HookList) { foreach ($h in $entry.hooks) { - if ($h.command -eq $command) { return } + if ($h.command -eq $Command) { + Write-Host "Already registered: $HookEvent $Matcher (skipping)" + return + } } } - [void]$preToolUse.Add(@{ - matcher = $matcher - hooks = @(@{ type = 'command'; command = $command }) + [void]$HookList.Add(@{ + matcher = $Matcher + hooks = @(@{ type = 'command'; command = $Command }) }) - Write-Host "Registered $matcher hook -> $command" + Write-Host "Registered $HookEvent $Matcher hook -> $Command" } -Add-MatcherHook -matcher 'Grep' -command $grepGuardCmd -Add-MatcherHook -matcher 'Agent' -command $preambleCmd -Add-MatcherHook -matcher 'WebSearch|WebFetch' -command $webGuardCmd +Add-MatcherHook -HookEvent 'PreToolUse' -HookList $preToolUse -Matcher 'Grep' -Command $grepGuardCmd +Add-MatcherHook -HookEvent 'PreToolUse' -HookList $preToolUse -Matcher 'Agent' -Command $preambleCmd +Add-MatcherHook -HookEvent 'PreToolUse' -HookList $preToolUse -Matcher 'WebSearch|WebFetch' -Command $webGuardCmd +Add-MatcherHook -HookEvent 'PreToolUse' -HookList $preToolUse -Matcher 'Edit|Write|MultiEdit' -Command $editGuardCmd +Add-MatcherHook -HookEvent 'PostToolUse' -HookList $postToolUse -Matcher 'mcp__codesearch__find_impact|mcp__codesearch__find' -Command $editGuardPostCmd -$settings['hooks']['PreToolUse'] = @($preToolUse) +$settings['hooks']['PreToolUse'] = @($preToolUse) +$settings['hooks']['PostToolUse'] = @($postToolUse) $settings | ConvertTo-Json -Depth 20 | Set-Content $settingsPath -Encoding utf8 diff --git a/integrations/claude-code/install.sh b/integrations/claude-code/install.sh index fd2a4b25..ae8ea764 100644 --- a/integrations/claude-code/install.sh +++ b/integrations/claude-code/install.sh @@ -33,11 +33,17 @@ mkdir -p "$HOOKS_DEST" cp "$HOOKS_SRC/grep-guard.sh" "$HOOKS_DEST/" cp "$HOOKS_SRC/subagent-preamble.sh" "$HOOKS_DEST/" cp "$HOOKS_SRC/web-guard.sh" "$HOOKS_DEST/" -chmod +x "$HOOKS_DEST/grep-guard.sh" "$HOOKS_DEST/subagent-preamble.sh" "$HOOKS_DEST/web-guard.sh" +cp "$HOOKS_SRC/edit-guard.sh" "$HOOKS_DEST/" +cp "$HOOKS_SRC/edit-guard-post.sh" "$HOOKS_DEST/" +cp "$HOOKS_SRC/codesearch-common.sh" "$HOOKS_DEST/" +chmod +x "$HOOKS_DEST/grep-guard.sh" "$HOOKS_DEST/subagent-preamble.sh" "$HOOKS_DEST/web-guard.sh" \ + "$HOOKS_DEST/edit-guard.sh" "$HOOKS_DEST/edit-guard-post.sh" GREP_GUARD_CMD="bash \"$HOOKS_DEST/grep-guard.sh\"" PREAMBLE_CMD="bash \"$HOOKS_DEST/subagent-preamble.sh\"" WEB_GUARD_CMD="bash \"$HOOKS_DEST/web-guard.sh\"" +EDIT_GUARD_CMD="bash \"$HOOKS_DEST/edit-guard.sh\"" +EDIT_GUARD_POST_CMD="bash \"$HOOKS_DEST/edit-guard-post.sh\"" mkdir -p "$CLAUDE_DIR" if [ -f "$SETTINGS_PATH" ]; then @@ -49,30 +55,33 @@ else settings='{}' fi -# Ensure hooks.PreToolUse exists as an array +# Ensure hooks.PreToolUse and hooks.PostToolUse exist as arrays settings=$(echo "$settings" | jq 'if has("hooks") then . else . + {hooks: {}} end - | .hooks |= (if has("PreToolUse") then . else . + {PreToolUse: []} end)') + | .hooks |= (if has("PreToolUse") then . else . + {PreToolUse: []} end) + | .hooks |= (if has("PostToolUse") then . else . + {PostToolUse: []} end)') already_registered() { - local cmd="$1" - echo "$settings" | jq -e --arg cmd "$cmd" \ - '.hooks.PreToolUse[]?.hooks[]? | select(.command == $cmd)' >/dev/null 2>&1 + local event="$1" cmd="$2" + echo "$settings" | jq -e --arg ev "$event" --arg cmd "$cmd" \ + '.hooks[$ev][]?.hooks[]? | select(.command == $cmd)' >/dev/null 2>&1 } add_matcher_hook() { - local matcher="$1" cmd="$2" - if already_registered "$cmd"; then - echo "Already registered: $matcher -> $cmd (skipping)" + local event="$1" matcher="$2" cmd="$3" + if already_registered "$event" "$cmd"; then + echo "Already registered: $event $matcher -> $cmd (skipping)" return fi - settings=$(echo "$settings" | jq --arg matcher "$matcher" --arg cmd "$cmd" \ - '.hooks.PreToolUse += [{matcher: $matcher, hooks: [{type: "command", command: $cmd}]}]') - echo "Registered $matcher hook -> $cmd" + settings=$(echo "$settings" | jq --arg ev "$event" --arg matcher "$matcher" --arg cmd "$cmd" \ + '.hooks[$ev] += [{matcher: $matcher, hooks: [{type: "command", command: $cmd}]}]') + echo "Registered $event $matcher hook -> $cmd" } -add_matcher_hook "Grep" "$GREP_GUARD_CMD" -add_matcher_hook "Agent" "$PREAMBLE_CMD" -add_matcher_hook "WebSearch|WebFetch" "$WEB_GUARD_CMD" +add_matcher_hook "PreToolUse" "Grep" "$GREP_GUARD_CMD" +add_matcher_hook "PreToolUse" "Agent" "$PREAMBLE_CMD" +add_matcher_hook "PreToolUse" "WebSearch|WebFetch" "$WEB_GUARD_CMD" +add_matcher_hook "PreToolUse" "Edit|Write|MultiEdit" "$EDIT_GUARD_CMD" +add_matcher_hook "PostToolUse" "mcp__codesearch__find_impact|mcp__codesearch__find" "$EDIT_GUARD_POST_CMD" echo "$settings" | jq '.' > "$SETTINGS_PATH" diff --git a/scripts/release-coauthors.sh b/scripts/release-coauthors.sh new file mode 100644 index 00000000..5035d21d --- /dev/null +++ b/scripts/release-coauthors.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Print `Co-authored-by:` trailers for everyone who authored work that a +# develop → master release squash is about to flatten away. +# +# Why: the contributors list is computed from the DEFAULT branch (master). +# Release PRs are squash-merged (one commit per release, authored by whoever +# clicks merge), so individual authorship would never reach master. GitHub +# counts `Co-authored-by:` trailers on the default branch, so carrying these +# trailers in the squash commit message credits every real contributor. +# +# Usage: +# scripts/release-coauthors.sh [BASE] [HEAD] # defaults: origin/master origin/develop +# +# Wire into the release (step 2 of RELEASING.md): +# gh pr merge --squash --admin \ +# --subject "release: v1.3.11" \ +# --body "$(scripts/release-coauthors.sh)" +# +# Bots and merge commits are excluded (merge commits carry no unique +# authorship). One line per distinct author email; the same human committing +# under two emails produces two lines — harmless. + +set -euo pipefail + +BASE="${1:-origin/master}" +HEAD="${2:-origin/develop}" + +emails=$(git log --no-merges --format='%an <%ae>' "$BASE..$HEAD" \ + | sort -u \ + | grep -viE 'github-actions\[bot\]|dependabot\[bot\]|41898282\+github-actions|test@example\.com' \ + || true) + +# Optional local blocklist (.docs/ is gitignored): one grep -E pattern per +# line, e.g. a device identity you never want credited. NEVER commit it. +BLOCKLIST=".docs/coauthors-blocklist" +if [ -f "$BLOCKLIST" ]; then + while IFS= read -r pattern; do + [ -z "$pattern" ] && continue + case "$pattern" in \#*) continue ;; esac + emails=$(printf '%s\n' "$emails" | grep -viE "$pattern" || true) + done < "$BLOCKLIST" +fi + +if [ -z "$emails" ]; then + echo "(no human commits between $BASE and $HEAD — nothing to credit)" >&2 + exit 0 +fi + +echo "$emails" | sed 's/^/Co-authored-by: /' +echo "(review the list before pasting — drop device/test identities you don't want credited)" >&2 diff --git a/scripts/test-githooks.sh b/scripts/test-githooks.sh new file mode 100755 index 00000000..7d508ec3 --- /dev/null +++ b/scripts/test-githooks.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# test-githooks.sh — pins the branch-aware agent-file lifecycle in .githooks/. +# +# Runs against the REAL .githooks/ files on a throwaway fixture repo (no cargo, +# no network): sources lib/root-md-guard.sh directly for the allowlist cases +# and invokes .githooks/post-checkout with hook-style arguments for the +# lifecycle cases. Run from anywhere: bash scripts/test-githooks.sh + +set -uo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +HOOKS="$REPO_ROOT/.githooks" +FIXTURE=$(mktemp -d) +PASS=0 +FAIL=0 + +cleanup() { rm -rf "$FIXTURE"; } +trap cleanup EXIT + +ok() { PASS=$((PASS + 1)); echo " PASS: $1"; } +bad() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; } + +assert() { # assert + if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (expected rc=$2, got rc=$3)"; fi +} + +run_guard() { # exit status of root_md_guard against the fixture's index + ( . "$HOOKS/lib/root-md-guard.sh" && root_md_guard ) >/dev/null 2>&1 +} + +stage() { git add -- "$@" >/dev/null 2>&1; } +unstage_all() { git reset >/dev/null 2>&1; } + +echo "== fixture ==" +cd "$FIXTURE" || exit 1 +# NB: cd + plain git (no `git -C`): native Windows git rejects MSYS /tmp paths +git init -q -b develop || { echo "git init failed"; exit 1; } +git config user.email t@t && git config user.name t +echo "# template" > AGENTS.develop.md +git add AGENTS.develop.md && git commit -qm init + +echo "== guard: develop blocks agent files, allows template ==" +echo plan > AGENTS.md && echo ptr > CLAUDE.md && echo diag > DIAGNOSE_x.md +stage AGENTS.md ; assert "develop: AGENTS.md introduction blocked" 1 "$(run_guard; echo $?)" +stage CLAUDE.md ; assert "develop: CLAUDE.md introduction blocked" 1 "$(run_guard; echo $?)" +stage DIAGNOSE_x.md ; assert "develop: stray md blocked" 1 "$(run_guard; echo $?)" +unstage_all +echo more >> AGENTS.develop.md && stage AGENTS.develop.md +assert "develop: AGENTS.develop.md allowed" 0 "$(run_guard; echo $?)" +unstage_all + +echo "== pre-merge-commit hook: executed, not just wired ==" +stage AGENTS.md +bash "$HOOKS/pre-merge-commit" >/dev/null 2>&1 +assert "pre-merge-commit blocks staged AGENTS.md on develop" 1 "$?" +unstage_all +bash "$HOOKS/pre-merge-commit" >/dev/null 2>&1 +assert "pre-merge-commit passes with a clean index on develop" 0 "$?" + +echo "== guard: feature branch allows agent files, blocks strays ==" +git checkout -qb feature/x +stage AGENTS.md CLAUDE.md +assert "feature: AGENTS.md+CLAUDE.md allowed" 0 "$(run_guard; echo $?)" +stage DIAGNOSE_x.md ; assert "feature: stray md blocked" 1 "$(run_guard; echo $?)" +unstage_all + +echo "== post-checkout: feature branch creates files when absent ==" +rm -f AGENTS.md CLAUDE.md +sh "$HOOKS/post-checkout" 0 0 1 >/dev/null +[ -f AGENTS.md ] && ok "feature checkout: AGENTS.md created" || bad "feature checkout: AGENTS.md created" +[ -f CLAUDE.md ] && ok "feature checkout: CLAUDE.md created" || bad "feature checkout: CLAUDE.md created" +[ "$(cat CLAUDE.md)" = "Read AGENTS.md." ] && ok "CLAUDE.md pointer content" || bad "CLAUDE.md pointer content" +printf 'workplan-marker\n' >> AGENTS.md + +echo "== post-checkout: never overwrites an existing work plan ==" +sh "$HOOKS/post-checkout" 0 0 1 >/dev/null +grep -q workplan-marker AGENTS.md && ok "existing AGENTS.md untouched" || bad "existing AGENTS.md untouched" + +echo "== post-checkout: develop removes untracked leftovers ==" +git checkout -q develop +[ -f AGENTS.md ] && ok "untracked AGENTS.md survived switch to develop (fixture premise)" \ + || bad "untracked AGENTS.md survived switch to develop (fixture premise)" +sh "$HOOKS/post-checkout" 0 0 1 >/dev/null +[ ! -f AGENTS.md ] && ok "develop: untracked AGENTS.md removed" || bad "develop: untracked AGENTS.md removed" +[ ! -f CLAUDE.md ] && ok "develop: untracked CLAUDE.md removed" || bad "develop: untracked CLAUDE.md removed" + +echo "== post-checkout: detached HEAD is a no-op ==" +git checkout -q --detach HEAD +sh "$HOOKS/post-checkout" 0 0 1 >/dev/null +[ ! -f AGENTS.md ] && [ ! -f CLAUDE.md ] && ok "detached: no files created" || bad "detached: no files created" + +echo "== wiring: pre-merge-commit gate exists and delegates to the guard ==" +[ -f "$HOOKS/pre-merge-commit" ] && ok "pre-merge-commit present" || bad "pre-merge-commit present" +grep -q "root-md-guard" "$HOOKS/pre-merge-commit" && ok "pre-merge-commit sources guard lib" || bad "pre-merge-commit sources guard lib" +grep -q "root-md-guard" "$HOOKS/pre-commit" && ok "pre-commit sources guard lib" || bad "pre-commit sources guard lib" +bash -n "$HOOKS/pre-commit" && bash -n "$HOOKS/pre-merge-commit" && bash -n "$HOOKS/lib/root-md-guard.sh" \ + && ok "hook syntax (bash -n)" || bad "hook syntax (bash -n)" + +echo "" +echo "githooks tests: $PASS passed, $FAIL failed" +[ "$FAIL" = 0 ] diff --git a/src/cache/file_meta.rs b/src/cache/file_meta.rs index 43991204..dab5f38e 100644 --- a/src/cache/file_meta.rs +++ b/src/cache/file_meta.rs @@ -343,7 +343,12 @@ impl FileMetaStore { let content = fs::read(path)?; let mut hasher = Sha256::new(); hasher.update(&content); - Ok(format!("{:x}", hasher.finalize())) + // sha2 0.11: the digest array no longer impls LowerHex — hex-encode manually + Ok(hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect()) } /// Get file modification time as unix timestamp diff --git a/src/chunker/mod.rs b/src/chunker/mod.rs index 972dd32b..f2bbbe1b 100644 --- a/src/chunker/mod.rs +++ b/src/chunker/mod.rs @@ -94,7 +94,12 @@ impl Chunk { pub fn compute_hash(content: &str) -> String { let mut hasher = Sha256::new(); hasher.update(content.as_bytes()); - format!("{:x}", hasher.finalize()) + // sha2 0.11: the digest array no longer impls LowerHex — hex-encode manually + hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect::() } /// TEST METHOD: Estimate memory usage of this chunk in bytes diff --git a/src/cli/claude_hooks.rs b/src/cli/claude_hooks.rs index 24a49626..8178640d 100644 --- a/src/cli/claude_hooks.rs +++ b/src/cli/claude_hooks.rs @@ -7,9 +7,11 @@ //! no dependency on the source tree at install time. //! //! Behaviour mirrors the shell installers: -//! 1. Write the hook scripts into `/hooks/codesearch/`. -//! 2. Merge one PreToolUse registration per guard into `settings.json`, -//! keyed by the exact command string so re-running never duplicates. +//! 1. Write the hook scripts (guards, the PostToolUse companion and the +//! shared helper libraries) into `/hooks/codesearch/`. +//! 2. Merge one PreToolUse registration per guard and one PostToolUse +//! registration per companion into `settings.json`, keyed by the exact +//! command string so re-running never duplicates. //! 3. Back up an existing `settings.json` before rewriting it. //! //! `` is `~/.claude` (user scope) or `./.claude` (`--project`). @@ -28,6 +30,16 @@ const PREAMBLE_PS1: &str = include_str!("../../integrations/claude-code/hooks/subagent-preamble.ps1"); const WEB_GUARD_SH: &str = include_str!("../../integrations/claude-code/hooks/web-guard.sh"); const WEB_GUARD_PS1: &str = include_str!("../../integrations/claude-code/hooks/web-guard.ps1"); +const EDIT_GUARD_SH: &str = include_str!("../../integrations/claude-code/hooks/edit-guard.sh"); +const EDIT_GUARD_PS1: &str = include_str!("../../integrations/claude-code/hooks/edit-guard.ps1"); +const EDIT_GUARD_POST_SH: &str = + include_str!("../../integrations/claude-code/hooks/edit-guard-post.sh"); +const EDIT_GUARD_POST_PS1: &str = + include_str!("../../integrations/claude-code/hooks/edit-guard-post.ps1"); +const CODESEARCH_COMMON_SH: &str = + include_str!("../../integrations/claude-code/hooks/codesearch-common.sh"); +const CODESEARCH_COMMON_PS1: &str = + include_str!("../../integrations/claude-code/hooks/codesearch-common.ps1"); /// A PreToolUse guard hook to install: the tool matcher it fires on, the script /// basename, and the per-platform script files to write out. @@ -44,6 +56,8 @@ struct GuardHook { /// - `Grep` → codesearch-first for internal code discovery. /// - `Agent` → inject a codesearch-first preamble into subagent prompts. /// - `WebSearch`/`WebFetch` → steer to remote doc mounts before the open web. +/// - `Edit`/`Write`/`MultiEdit` → require a recent codesearch consultation +/// (find_impact / find kind="usages") per file before edits. static GUARD_HOOKS: &[GuardHook] = &[ GuardHook { matcher: "Grep", @@ -70,6 +84,34 @@ static GUARD_HOOKS: &[GuardHook] = &[ ("web-guard.ps1", WEB_GUARD_PS1), ], }, + GuardHook { + matcher: "Edit|Write|MultiEdit", + stem: "edit-guard", + files: &[ + ("edit-guard.sh", EDIT_GUARD_SH), + ("edit-guard.ps1", EDIT_GUARD_PS1), + ], + }, +]; + +/// PostToolUse companion hooks. These can never block anything (Claude Code +/// ignores decisions there); they only record state — `edit-guard-post` +/// marks "codesearch consulted" per file path after qualifying MCP calls, +/// which `edit-guard` honours on the next Edit/Write/MultiEdit. +static POST_TOOL_USE_HOOKS: &[GuardHook] = &[GuardHook { + matcher: "mcp__codesearch__find_impact|mcp__codesearch__find", + stem: "edit-guard-post", + files: &[ + ("edit-guard-post.sh", EDIT_GUARD_POST_SH), + ("edit-guard-post.ps1", EDIT_GUARD_POST_PS1), + ], +}]; + +/// Shared helper libraries sourced by the guard scripts at runtime; written +/// once into the same hooks directory as the guards themselves. +static COMMON_HOOK_FILES: &[(&str, &str)] = &[ + ("codesearch-common.sh", CODESEARCH_COMMON_SH), + ("codesearch-common.ps1", CODESEARCH_COMMON_PS1), ]; /// Resolve the `.claude` directory for the requested scope. @@ -97,12 +139,17 @@ fn hook_command(hooks_dest: &Path, stem: &str) -> String { } } -/// Ensure `settings.hooks.PreToolUse` contains a `{matcher, hooks:[…]}` entry +/// Ensure `settings.hooks.` contains a `{matcher, hooks:[…]}` entry /// for `command`. Idempotent: returns `Ok(false)` without modifying anything if -/// an entry with this exact `command` already exists anywhere in `PreToolUse`. -/// Returns an error if a pre-existing `hooks`/`PreToolUse` value has a shape +/// an entry with this exact `command` already exists anywhere in ``. +/// Returns an error if a pre-existing `hooks`/`` value has a shape /// incompatible with the expected object/array. -fn add_matcher_hook(settings: &mut Value, matcher: &str, command: &str) -> Result { +fn add_matcher_hook( + settings: &mut Value, + event: &str, + matcher: &str, + command: &str, +) -> Result { let root = settings .as_object_mut() .context("settings.json root must be a JSON object")?; @@ -111,13 +158,13 @@ fn add_matcher_hook(settings: &mut Value, matcher: &str, command: &str) -> Resul .or_insert_with(|| json!({})) .as_object_mut() .context("settings.hooks must be a JSON object")?; - let pre = hooks - .entry("PreToolUse") + let list = hooks + .entry(event) .or_insert_with(|| json!([])) .as_array_mut() - .context("settings.hooks.PreToolUse must be a JSON array")?; + .context("settings.hooks.{event} must be a JSON array")?; - let already = pre.iter().any(|entry| { + let already = list.iter().any(|entry| { entry .get("hooks") .and_then(Value::as_array) @@ -131,13 +178,28 @@ fn add_matcher_hook(settings: &mut Value, matcher: &str, command: &str) -> Resul return Ok(false); } - pre.push(json!({ + list.push(json!({ "matcher": matcher, "hooks": [ { "type": "command", "command": command } ] })); Ok(true) } +/// Write one hook script (or shared library) into the hooks directory, +/// executable on unix. Scripts are re-written verbatim on every install. +fn write_hook_file(hooks_dest: &Path, name: &str, contents: &str) -> Result<()> { + let path = hooks_dest.join(name); + std::fs::write(&path, contents).with_context(|| format!("writing {}", path.display()))?; + #[cfg(unix)] + if name.ends_with(".sh") { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&path)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms)?; + } + Ok(()) +} + /// Load an existing `settings.json` (backing it up first) or start from `{}`. fn load_or_init_settings(settings_path: &Path) -> Result { if !settings_path.exists() { @@ -163,36 +225,36 @@ pub fn run_claude_install(project: bool) -> Result<()> { std::fs::create_dir_all(&hooks_dest) .with_context(|| format!("creating {}", hooks_dest.display()))?; - // 1. Write the hook scripts. - for gh in GUARD_HOOKS { + // 1. Write the hook scripts (guards, PostToolUse companions, shared + // helper libraries — all into the same directory). + for (name, contents) in COMMON_HOOK_FILES { + write_hook_file(&hooks_dest, name, contents)?; + } + for gh in GUARD_HOOKS.iter().chain(POST_TOOL_USE_HOOKS) { for (name, contents) in gh.files { - let path = hooks_dest.join(name); - std::fs::write(&path, contents) - .with_context(|| format!("writing {}", path.display()))?; - #[cfg(unix)] - if name.ends_with(".sh") { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&path)?.permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&path, perms)?; - } + write_hook_file(&hooks_dest, name, contents)?; } } - // 2. Merge the PreToolUse registrations into settings.json. + // 2. Merge the PreToolUse / PostToolUse registrations into settings.json. // (`claude_dir` already exists — creating `hooks_dest` above made it.) let settings_path = claude_dir.join("settings.json"); let mut settings = load_or_init_settings(&settings_path)?; - for gh in GUARD_HOOKS { - let cmd = hook_command(&hooks_dest, gh.stem); - if add_matcher_hook(&mut settings, gh.matcher, &cmd)? { - eprintln!( - "{}", - format!("Registered {} hook -> {}", gh.matcher, cmd).green() - ); - } else { - eprintln!("Already registered: {} (skipping)", gh.matcher); + for (event, hooks) in [ + ("PreToolUse", GUARD_HOOKS), + ("PostToolUse", POST_TOOL_USE_HOOKS), + ] { + for gh in hooks { + let cmd = hook_command(&hooks_dest, gh.stem); + if add_matcher_hook(&mut settings, event, gh.matcher, &cmd)? { + eprintln!( + "{}", + format!("Registered {} {} hook -> {}", event, gh.matcher, cmd).green() + ); + } else { + eprintln!("Already registered: {} {} (skipping)", event, gh.matcher); + } } } @@ -217,7 +279,8 @@ mod tests { #[test] fn add_matcher_hook_registers_on_empty_settings() { let mut settings = json!({}); - let added = add_matcher_hook(&mut settings, "Grep", "bash /x/grep-guard.sh").unwrap(); + let added = + add_matcher_hook(&mut settings, "PreToolUse", "Grep", "bash /x/grep-guard.sh").unwrap(); assert!(added, "first registration must add the hook"); let pre = settings["hooks"]["PreToolUse"].as_array().unwrap(); assert_eq!(pre.len(), 1); @@ -230,12 +293,41 @@ mod tests { fn add_matcher_hook_is_idempotent_by_command() { let mut settings = json!({}); let cmd = "bash /x/grep-guard.sh"; - assert!(add_matcher_hook(&mut settings, "Grep", cmd).unwrap()); + assert!(add_matcher_hook(&mut settings, "PreToolUse", "Grep", cmd).unwrap()); // Same command again -> no-op, no duplicate. - assert!(!add_matcher_hook(&mut settings, "Grep", cmd).unwrap()); + assert!(!add_matcher_hook(&mut settings, "PreToolUse", "Grep", cmd).unwrap()); assert_eq!(settings["hooks"]["PreToolUse"].as_array().unwrap().len(), 1); } + #[test] + fn post_tool_use_hook_is_idempotent_by_command() { + let mut settings = json!({}); + let cmd = "bash /x/edit-guard-post.sh"; + assert!(add_matcher_hook( + &mut settings, + "PostToolUse", + "mcp__codesearch__find_impact|mcp__codesearch__find", + cmd + ) + .unwrap()); + // Same command again -> no-op, no duplicate, and the entry stays in + // PostToolUse (never leaking into PreToolUse). + assert!(!add_matcher_hook( + &mut settings, + "PostToolUse", + "mcp__codesearch__find_impact|mcp__codesearch__find", + cmd + ) + .unwrap()); + let post = settings["hooks"]["PostToolUse"].as_array().unwrap(); + assert_eq!(post.len(), 1); + assert_eq!( + post[0]["matcher"], + "mcp__codesearch__find_impact|mcp__codesearch__find" + ); + assert!(settings["hooks"].get("PreToolUse").is_none()); + } + #[test] fn add_matcher_hook_preserves_unrelated_entries() { let mut settings = json!({ @@ -246,7 +338,9 @@ mod tests { ] } }); - assert!(add_matcher_hook(&mut settings, "Grep", "bash /x/grep-guard.sh").unwrap()); + assert!( + add_matcher_hook(&mut settings, "PreToolUse", "Grep", "bash /x/grep-guard.sh").unwrap() + ); let pre = settings["hooks"]["PreToolUse"].as_array().unwrap(); assert_eq!(pre.len(), 2, "existing Bash hook must survive"); assert_eq!(settings["model"], "opus", "unrelated settings must survive"); @@ -255,29 +349,38 @@ mod tests { #[test] fn add_matcher_hook_rejects_bad_pretooluse_shape() { let mut settings = json!({ "hooks": { "PreToolUse": "not-an-array" } }); - assert!(add_matcher_hook(&mut settings, "Grep", "cmd").is_err()); + assert!(add_matcher_hook(&mut settings, "PreToolUse", "Grep", "cmd").is_err()); } #[test] fn add_matcher_hook_rejects_non_object_hooks() { let mut settings = json!({ "hooks": "not-an-object" }); - assert!(add_matcher_hook(&mut settings, "Grep", "cmd").is_err()); + assert!(add_matcher_hook(&mut settings, "PreToolUse", "Grep", "cmd").is_err()); } #[test] fn add_matcher_hook_rejects_non_object_root() { let mut settings = json!(["not", "an", "object"]); - assert!(add_matcher_hook(&mut settings, "Grep", "cmd").is_err()); + assert!(add_matcher_hook(&mut settings, "PreToolUse", "Grep", "cmd").is_err()); } #[test] - fn guard_hooks_cover_grep_agent_and_web() { + fn guard_hooks_cover_grep_agent_edit_and_web() { let matchers: Vec<&str> = GUARD_HOOKS.iter().map(|g| g.matcher).collect(); assert!(matchers.contains(&"Grep")); assert!(matchers.contains(&"Agent")); assert!(matchers.contains(&"WebSearch|WebFetch")); - // Every guard ships both a .sh and a .ps1 with non-empty embedded bodies. - for g in GUARD_HOOKS { + assert!(matchers.contains(&"Edit|Write|MultiEdit")); + // The PostToolUse companion registers under its own event list. + let post: Vec<&str> = POST_TOOL_USE_HOOKS.iter().map(|g| g.matcher).collect(); + assert!(post.contains(&"mcp__codesearch__find_impact|mcp__codesearch__find")); + // The shared helper libraries ship alongside the guards. + for (name, body) in COMMON_HOOK_FILES { + assert!(!body.is_empty(), "{name} embedded body is empty"); + } + // Every guard and companion ships both a .sh and a .ps1 with + // non-empty embedded bodies. + for g in GUARD_HOOKS.iter().chain(POST_TOOL_USE_HOOKS) { assert_eq!( g.files.len(), 2, diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index 7525c0ee..2cb3f413 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -893,6 +893,7 @@ fn print_results(results: &[CheckResult], json: bool) { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; use std::fs::{self, File}; use std::io::Write; use tempfile::tempdir; @@ -946,7 +947,11 @@ mod tests { create_fts_dir(dir); } + /// Mutates CODESEARCH_REPOS_CONFIG — `#[serial]` + the EnvRestore guard + /// keep it from racing (and leaking into) the remove_order_tests, which + /// redirect the same var to their own fixtures. #[test] + #[serial] fn test_doctor_no_database() { let temp_dir = tempdir().unwrap(); let project_path = temp_dir.path(); @@ -954,19 +959,21 @@ mod tests { // Isolate from global repos.json — point to non-existent config so // find_best_database doesn't discover the developer's real database. let fake_config = temp_dir.path().join("nonexistent_repos.json"); - std::env::set_var(crate::constants::REPOS_CONFIG_ENV, &fake_config); + let _env = crate::testing::EnvRestore::set(&[( + crate::constants::REPOS_CONFIG_ENV, + &fake_config.to_string_lossy(), + )]); // No .codesearch.db exists let result = check_find_database(project_path); - std::env::remove_var(crate::constants::REPOS_CONFIG_ENV); - assert_eq!(result.status, CheckStatus::Fail); assert_eq!(result.name, "No database found"); assert!(result.message.contains("No .codesearch.db found")); } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_incomplete_database() { let temp_dir = tempdir().unwrap(); let db_dir = temp_dir.path().join(".codesearch.db"); @@ -983,6 +990,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_model_name_mismatch() { let temp_dir = tempdir().unwrap(); let db_dir = temp_dir.path().join(".codesearch.db"); @@ -1001,6 +1009,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_model_name_consistent() { let temp_dir = tempdir().unwrap(); let db_dir = temp_dir.path().join(".codesearch.db"); @@ -1018,6 +1027,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_misplaced_index() { let temp_dir = tempdir().unwrap(); @@ -1038,6 +1048,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_index_at_git_root() { let temp_dir = tempdir().unwrap(); @@ -1057,6 +1068,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_stale_files() { let temp_dir = tempdir().unwrap(); let project_path = temp_dir.path(); @@ -1098,6 +1110,7 @@ mod tests { } #[test] + #[serial] // reads global repos.json via find_best_database fn test_doctor_valid_database_all_green() { let temp_dir = tempdir().unwrap(); let db_dir = temp_dir.path().join(".codesearch.db"); diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 3e640e9e..0b64ced8 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -261,7 +261,9 @@ pub struct Cli { #[arg(long, global = true)] pub store: Option, - /// Embedding model to use (e.g., bge-small, jina-code, embeddinggemma-q4) + /// Embedding model to use (e.g., bge-small, jina-code, embeddinggemma-q4). + /// On `serve`, this is the default for newly created indexes; each repo's + /// existing index keeps the model recorded in its own metadata. #[arg(long, global = true)] pub model: Option, } @@ -517,6 +519,24 @@ pub enum Commands { )] create_index: bool, + /// Serve a caller-owned local index without writing to it. + /// + /// Opens the database read-only and never runs incremental refresh or + /// file watching. On its own this still serves an index another process + /// is building; pair it with --require-ready to reject that case. + /// Requires --mode local. + #[arg(long)] + readonly: bool, + + /// Refuse to start unless the existing local index is complete. + /// + /// Verifies the vector and full-text stores hold data before the MCP + /// transport opens, so an incomplete index fails startup instead of + /// answering `status` with "building" and `search` with no results. + /// Requires --mode local. + #[arg(long)] + require_ready: bool, + /// MCP connection mode (default: auto, override with CODESEARCH_MCP_MODE) /// /// - auto: Connect to serve if running, otherwise use local DB @@ -1204,10 +1224,19 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Err(e) = crate::logger::init_serve_logger(log_level, effective_quiet) { eprintln!("Warning: failed to initialize serve logger: {}", e); } + // `--model` is a global flag inherited by every subcommand. On + // `serve` it sets the serve-wide default for newly created + // indexes — each repo's queries still use the model recorded + // in its own index metadata (a hub may mix models), so this + // never overrides an existing index. + if let Some(mt) = model_type { + warn_if_heavier_model(mt); + } crate::serve::run_serve( host, port, register, + model_type, no_tui, keep_warm_url, idle_suspend_secs, @@ -1228,6 +1257,8 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { Commands::Mcp { path, create_index, + readonly, + require_ready, mode, } => { // Logger is initialized inside run_mcp_server() once db_path is known. @@ -1235,8 +1266,24 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { // // MCP stdio transport uses stdout for JSON-RPC — always force file-only // logging to keep the channel clean, regardless of the global --quiet flag. - crate::mcp::run_mcp_server(path, create_index, log_level, true, mode, cancel_token) + if readonly || require_ready { + crate::mcp::run_mcp_server_with_options( + path, + crate::mcp::McpStartupOptions { + create_index, + readonly, + require_ready, + }, + log_level, + true, + mode, + cancel_token, + ) .await + } else { + crate::mcp::run_mcp_server(path, create_index, log_level, true, mode, cancel_token) + .await + } } Commands::Cache { command } => match command { CacheCommands::Stats { model } => run_cache_stats(model).await, @@ -2050,6 +2097,61 @@ mod tests { } } + #[test] + fn test_mcp_readonly_and_require_ready_default_to_false() { + let cli = Cli::try_parse_from(["codesearch", "mcp"]).expect("cli parse should succeed"); + match cli.command { + Commands::Mcp { + readonly, + require_ready, + .. + } => { + assert!(!readonly); + assert!(!require_ready); + } + _ => panic!("expected Mcp command"), + } + } + + #[test] + fn test_mcp_readonly_does_not_imply_other_flags() { + let cli = Cli::try_parse_from(["codesearch", "mcp", "--readonly"]) + .expect("cli parse should succeed"); + match cli.command { + Commands::Mcp { + readonly, + require_ready, + create_index, + .. + } => { + assert!(readonly); + assert!(!require_ready, "--readonly must not imply --require-ready"); + assert!( + create_index, + "--readonly must not imply --create-index=false" + ); + } + _ => panic!("expected Mcp command"), + } + } + + #[test] + fn test_mcp_require_ready_does_not_imply_readonly() { + let cli = Cli::try_parse_from(["codesearch", "mcp", "--require-ready"]) + .expect("cli parse should succeed"); + match cli.command { + Commands::Mcp { + require_ready, + readonly, + .. + } => { + assert!(require_ready); + assert!(!readonly, "--require-ready must not imply --readonly"); + } + _ => panic!("expected Mcp command"), + } + } + #[test] fn test_cli_no_repos_subcommand() { let result = Cli::try_parse_from(["codesearch", "repos", "--help"]); diff --git a/src/constants.rs b/src/constants.rs index d1514472..f0bd457e 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -265,8 +265,8 @@ pub fn resolve_serve_host() -> String { } /// Environment variable to set the admin API key for management endpoints. -/// When set, all management routes (`POST /repos`, `DELETE /repos/:alias`, -/// `POST /repos/:alias/reindex`, `POST /reload`) require this key. +/// When set, all management routes (`POST /repos`, `DELETE /repos/{alias}`, +/// `POST /repos/{alias}/reindex`, `POST /reload`) require this key. /// When unset or empty, management routes are unauthenticated (backward compatible). /// The key is validated against `Authorization: Bearer ` or `X-API-Key: ` headers. pub const SERVE_API_KEY_ENV: &str = "CODESEARCH_SERVE_API_KEY"; @@ -349,6 +349,20 @@ pub const MCP_ENDPOINT_PATH: &str = "/mcp"; /// Returns JSON snapshot of all repo states, sessions, and CPU usage. pub const STATUS_PATH: &str = "/status"; +/// Indexing-freshness endpoint path served by `codesearch serve`. +/// +/// Cheap single-question probe for the grep-guard hook (and any caller that +/// needs to distinguish "no results" from "index mid-rebuild"): takes +/// `?path=`, resolves the containing registered repo, and +/// returns `{"covered":bool,"alias":..,"indexing":bool}` — `indexing` is true +/// while that repo has an active (non-stale) reindex in flight, which +/// includes the full refresh fired by a branch switch. Same auth class as +/// [`STATUS_PATH`]: reachable without the admin key on localhost, protected +/// by `require_auth_for_network` on network binds. `/healthz` remains the +/// ONLY always-unauthenticated endpoint — liveness and freshness are +/// different questions and stay on different paths. +pub const INDEXING_PATH: &str = "/indexing"; + /// Remotes endpoint path served by `codesearch serve`. /// /// Observability companion to [`STATUS_PATH`]: lists the configured federation @@ -373,8 +387,13 @@ pub const FIND_PATH: &str = "/find"; pub const EXPLORE_PATH: &str = "/explore"; /// REST get-chunk endpoint (HTTP mirror of the `get_chunk` MCP tool). -/// GET `/chunk/:id?context_lines=&project=&group=`. -pub const CHUNK_PATH: &str = "/chunk/:id"; +/// GET `/chunk/{id}?context_lines=&project=&group=`. +pub const CHUNK_PATH: &str = "/chunk/{id}"; + +/// REST find-impact endpoint (HTTP mirror of the `find_impact` MCP tool). +/// POST a `FindImpactRequest` body; returns the tool's JSON payload +/// (busy envelope and index-freshness fields included). +pub const FIND_IMPACT_PATH: &str = "/find-impact"; /// How long an open repo may remain idle (no queries) before it is evicted. /// Eviction closes the DB handles, stops the FSW, and releases memory. @@ -494,6 +513,31 @@ pub const MAX_INDEXING_SECS: u64 = 30 * 60; // 30 minutes /// Environment variable to override the maximum indexing duration. pub const MAX_INDEXING_SECS_ENV: &str = "CODESEARCH_MAX_INDEXING_SECS"; +/// Total number of attempts (initial request + retries) the federation client +/// makes against a remote peer that answers with a transient HTTP status +/// (502/503/504). Federated peers commonly run on scale-to-zero hosts (Azure +/// Container Apps): the first request after an idle period can hit a cold +/// start and surface as a 503 even though the peer is perfectly healthy. A +/// short bounded retry inside the active tool call absorbs most cold starts +/// before the caller ever sees them. This is NOT a poll: retries only happen +/// while a user-initiated tool call is already in flight against that peer +/// (the "never contact a federated peer on a cadence" rule in AGENTS.md is +/// about timers, and stays intact). +pub const REMOTE_PEER_RETRY_ATTEMPTS: u32 = 3; + +/// Backoff (milliseconds) between federation retry attempts, one entry per +/// retry (so `REMOTE_PEER_RETRY_ATTEMPTS - 1` entries; the last entry is +/// reused if there are ever more retries than entries). Short by design — +/// the retry exists to catch a peer that is already warming, not to outwait +/// a long deployment. If the peer is still transient-failing after the last +/// attempt, the error message tells the caller to retry the same call in +/// ~30s instead of blocking the tool call longer. +pub const REMOTE_PEER_RETRY_BACKOFF_MS: &[u64] = &[3000, 8000]; + +/// Environment variable overriding every federation retry backoff with a +/// single millisecond value (test hook so retry tests don't sleep for real). +pub const REMOTE_PEER_RETRY_BACKOFF_ENV: &str = "CODESEARCH_REMOTE_RETRY_BACKOFF_MS"; + /// Cooperative join window (seconds) for `await_index_task` and /// `await_fsw_shutdown`: how long a background indexing / file-watcher task /// is given to observe its `CancellationToken` and exit on its own before it @@ -518,6 +562,26 @@ pub const DB_DELETE_RETRY_INITIAL_MS: u64 = 200; /// delete retries in `remove_repo`. pub const DB_DELETE_RETRY_BACKOFF_CAP_MS: u64 = 2000; +/// Poll interval (milliseconds) for the in-process LMDB-holder release wait +/// inside `remove_repo`'s locked-DB delete retry loop. After a lock-class +/// delete failure the loop polls `lmdb_registry::open_holders_under` at this +/// cadence until every in-process env under the DB dir is released (or +/// `DB_DELETE_RETRY_BUDGET_SECS` expires), so the next attempt runs against +/// an actually-unlocked directory instead of burning attempts blind. +pub const DB_DELETE_ENV_RELEASE_POLL_MS: u64 = 100; + +/// Unallocated margin (seconds) the CLI's delegated `DELETE /repos/{alias}` +/// request adds on top of serve's legitimate worst-case removal time — +/// `DB_DELETE_RETRY_BUDGET_SECS` plus one `BG_TASK_COOPERATIVE_TIMEOUT_SECS` +/// per cooperative join (FSW task + index task) — so the CLI receives +/// serve's honest locked-DB outcome (`db_deleted` / payload) instead of its +/// own request timeout firing first. The shared delegation client's 3 s +/// total timeout is fine for the `/health` probe but far shorter than a +/// legitimate slow removal (warmup cancellation + env-release wait + +/// retries); `try_delegate_rm_to_serve` builds the DELETE its own client +/// sized from these constants. +pub const RM_DELEGATE_DELETE_MARGIN_SECS: u64 = 10; + /// Default embedding dimensions used when metadata is missing or unreadable. pub const DEFAULT_EMBEDDING_DIMENSIONS: usize = 384; @@ -552,9 +616,30 @@ pub const SCIP_CSHARP_DEBOUNCE_MS: u64 = 60_000; // 60 seconds /// LMDB database name for the SCIP symbols table. pub const SCIP_SYMBOLS_DB_NAME: &str = "scip_symbols"; +/// LMDB database name for the SCIP per-repo metadata table. +pub const SCIP_META_DB_NAME: &str = "scip_meta"; + /// LMDB metadata key for the last rebuild timestamp. pub const SCIP_REBUILD_TIMESTAMP_KEY: &str = "last_rebuild_ts"; +/// LMDB metadata key for the git HEAD sha the symbol index was built for. +/// Written on rebuild when the repo HEAD is readable; absent means unknown +/// (never written, or git could not be read at build time). +pub const SCIP_HEAD_SHA_KEY: &str = "head_sha"; + +/// LMDB metadata key recording which symbol-key format generation an index +/// was built with. Written at every C# SCIP rebuild; `has_index` refuses an +/// index whose value is absent or differs from [`SCIP_KEY_FORMAT`], so a +/// change to the canonical key format forces exactly one rebuild instead of +/// old-format keys being served as fresh. +pub const SCIP_KEY_FORMAT_KEY: &str = "key_format"; + +/// Current value written for [`SCIP_KEY_FORMAT_KEY`]. Bump whenever the +/// canonical SCIP symbol key format produced by a language helper changes +/// shape (B4: C# generic arity / containing-type path / fully qualified +/// parameter types). +pub const SCIP_KEY_FORMAT: &str = "2"; + /// LMDB table mapping `(file:line)` positions to `[symbol_keys]`. /// Used for O(1) position-based symbol lookup. pub const SCIP_POSITION_DB_NAME: &str = "scip_positions"; @@ -569,6 +654,15 @@ pub const SCIP_SIMPLE_NAMES_DB_NAME: &str = "scip_simple_names"; /// cleared when the definition index is rebuilt. Gives O(1) lookup on 2nd+ calls. pub const SCIP_REF_CACHE_DB_NAME: &str = "scip_ref_cache"; +/// LMDB table caching per-symbol completeness warnings from on-demand +/// reference resolution (`scip-csharp find-refs` / `batch-find-refs`). +/// Key: full SCIP symbol key. Value: `[v1, bincode(Vec)]` (same wire +/// format as the key lists). Non-empty means the cached references may be +/// INCOMPLETE — a helper failure was survived rather than fatal. Empty +/// warnings are REMOVED rather than stored, so absence means "complete" and +/// a later clean re-resolution clears a stale warning. +pub const SCIP_REF_WARNINGS_DB_NAME: &str = "scip_ref_warnings"; + /// Language identifier for the C# symbol indexer. /// Used as a key in `SymbolIndexerRegistry` lookups and TUI status maps. pub const LANG_CSHARP: &str = "csharp"; @@ -586,6 +680,18 @@ pub const SCIP_TYPESCRIPT_HELPER_ENV: &str = "CODESEARCH_SCIP_TYPESCRIPT"; /// so both adapters can safely share the same `scip_meta` table if ever merged. pub const SCIP_TYPESCRIPT_REBUILD_TIMESTAMP_KEY: &str = "last_rebuild_ts:typescript"; +/// TypeScript-specific key for `SCIP_HEAD_SHA_KEY` (the C# and TypeScript +/// adapters share one `scip_meta` table, so keys are language-prefixed). +pub const SCIP_TYPESCRIPT_HEAD_SHA_KEY: &str = "head_sha:typescript"; + +/// TypeScript-specific key for the index-warnings entry: a JSON array of +/// warnings from the LAST `scip-typescript` run. Non-empty means the index +/// may be incomplete (the run exited non-zero and partial output was kept); +/// an empty array is written on every clean rebuild so a successful reindex +/// clears stale warnings. Absence (indexes written before this key existed) +/// is read as complete — there is nothing to claim otherwise. +pub const SCIP_TYPESCRIPT_INDEX_WARNINGS_KEY: &str = "index_warnings:typescript"; + /// Debounce window (ms) for the TypeScript file-watcher symbol rebuild. /// Mirrors `SCIP_CSHARP_DEBOUNCE_MS` — a single quiet-period flush avoids /// spawning `scip-typescript` once per saved file during a burst of edits. @@ -645,6 +751,67 @@ pub const SCIP_LMDB_DEFAULT_MAP_SIZE_MB: usize = 512; /// When set, takes precedence over `SCIP_LMDB_DEFAULT_MAP_SIZE_MB`. pub const SCIP_LMDB_MAP_SIZE_MB_ENV: &str = "CODESEARCH_SCIP_LMDB_MAP_MB"; +/// Internal wall-clock budget (seconds) for a single `find_impact` reference +/// lookup. +/// +/// A cold reference-cache miss makes the `find_impact` handler invoke the +/// external SCIP helper (`scip-csharp find-refs`), which can take several +/// minutes on a large solution. Without an internal deadline the MCP client +/// is the timeout mechanism: it aborts with an opaque `-32001 Request timed +/// out` and the calling agent falls back to plain-text search exactly when +/// the precise SCIP call graph is most useful. This budget makes the server +/// answer first with a structured busy envelope +/// (`{"busy": true, "state": ..., "waited_ms": ..., "advice": ...}`) while +/// the lookup continues in the background, so a retry is served warm from +/// the reference cache instead of cold again. +/// +/// Override at runtime with `CODESEARCH_FIND_IMPACT_BUDGET_SECS` (integer +/// seconds). `0` disables the budget entirely, restoring the previous +/// unbounded-blocking behaviour. Unparseable values fall back to the default. +/// +/// The default is deliberately BELOW typical MCP client timeouts (observed +/// live: an MCP client gave up at ~60s with `-32001` while the busy answer +/// was still being prepared at the 60s budget) — the structured busy answer +/// is only useful if it arrives before the client stops listening. +pub const DEFAULT_FIND_IMPACT_BUDGET_SECS: u64 = 45; + +/// Environment variable to override `DEFAULT_FIND_IMPACT_BUDGET_SECS`. +pub const FIND_IMPACT_BUDGET_SECS_ENV: &str = "CODESEARCH_FIND_IMPACT_BUDGET_SECS"; + +/// Maximum number of resident SCIP helper workspaces (todo #115). +/// +/// Admission control IS the memory governor: each resident workspace holds a +/// fully loaded Roslyn solution (1-2 GB on large solutions), so the pool cap +/// bounds total helper memory to `MAX_RESIDENT x heap cap`. A third repo's +/// lookup evicts the least-recently-used workspace — eviction is safe because +/// resolved references persist in the LMDB ref cache, so only latency is +/// lost, never data. +pub const DEFAULT_SCIP_MAX_RESIDENT_WORKSPACES: usize = 2; +pub const SCIP_MAX_RESIDENT_WORKSPACES_ENV: &str = "CODESEARCH_SCIP_MAX_RESIDENT"; + +/// Per-workspace managed-heap cap passed to the helper as +/// `DOTNET_GCHeapHardLimit` (bytes; the env var is interpreted as hex by the +/// .NET runtime, so the Rust side formats it without a prefix). A runaway +/// workspace fails fast at the cap instead of taking the machine with it — +/// the typed `failed` path turns that into an agent-actionable answer. +pub const DEFAULT_SCIP_WORKSPACE_HEAP_CAP: u64 = 1_610_612_736; // 1.5 GiB +pub const SCIP_WORKSPACE_HEAP_CAP_ENV: &str = "CODESEARCH_SCIP_WORKSPACE_HEAP_CAP"; + +/// Resident workspaces idle longer than this are torn down by lazy reaping +/// (checked on pool access). Restarting a workspace costs one solution load — +/// acceptable for an idle repo, which is exactly what the TTL measures. +pub const DEFAULT_SCIP_WORKSPACE_IDLE_SECS: u64 = 600; +pub const SCIP_WORKSPACE_IDLE_SECS_ENV: &str = "CODESEARCH_SCIP_WORKSPACE_IDLE_SECS"; + +/// How long (seconds) a budget-overrun `find_impact` lookup stays tracked +/// for retry observation. Must comfortably exceed the slowest legitimate +/// `scip-csharp find-refs` run (several minutes on a large solution): an +/// entry dropped while its lookup is still running would turn a retry into +/// a cold restart, voiding the dedupe the busy advice promises. Finished +/// entries are removed on their first retry read, so this cap only bounds +/// abandoned lookups. +pub const FIND_IMPACT_TRACK_TTL_SECS: u64 = 1800; + /// Debounce window (seconds) for persisting repos.json metadata updates. /// Coalesces bursts of file changes into a single write. pub const PERSIST_DEBOUNCE_SECS: u64 = 10; diff --git a/src/db_discovery/mod.rs b/src/db_discovery/mod.rs index fed6e979..8b12eef0 100644 --- a/src/db_discovery/mod.rs +++ b/src/db_discovery/mod.rs @@ -278,7 +278,14 @@ pub fn register_repository(project_path: &Path) -> Result<()> { Ok(()) } -/// Unregister a repository from global tracking +/// Unregister a repository from global tracking. +/// +/// No longer called from `remove_from_index` (todo #48 fix folded this +/// load/unregister_path/save sequence inline there so the file-delete could +/// run first and the config mutation only happen after it succeeded) — kept +/// `pub` for CLI/admin tooling that wants a plain unregister with no +/// file-removal side effect. +#[allow(dead_code)] // Available for CLI and admin tooling, see doc comment above pub fn unregister_repository(project_path: &Path) -> Result<()> { let mut config = ReposConfig::load()?; if config.unregister_path(project_path) { diff --git a/src/db_discovery/repos.rs b/src/db_discovery/repos.rs index b2b27b94..36ce1bd2 100644 --- a/src/db_discovery/repos.rs +++ b/src/db_discovery/repos.rs @@ -316,17 +316,92 @@ impl ReposConfig { } pub fn save(&self) -> Result<()> { + // Under cargo test, writing the REAL global repos.json is always a + // bug: every test must point CODESEARCH_REPOS_CONFIG at a temp file + // before anything on the save path runs. This guard exists because + // its absence once let a mis-ordered test seed overwrite a + // developer's entire registry (17 repos, groups, remotes — all of + // it) with a one-entry fixture, recovered only from serve logs. + #[cfg(test)] + { + let override_set = std::env::var(crate::constants::REPOS_CONFIG_ENV) + .map(|v| !v.trim().is_empty()) + .unwrap_or(false); + let escape = std::env::var("CODESEARCH_TEST_ALLOW_GLOBAL_SAVE").is_ok(); + if !override_set && !escape { + panic!( + "ReposConfig::save() under cargo test would write the real global \ + repos.json. Set CODESEARCH_REPOS_CONFIG to a temp path first (or set \ + CODESEARCH_TEST_ALLOW_GLOBAL_SAVE if this is genuinely intended)." + ); + } + } let path = Self::path()?; self.save_to(&path) } /// Save to an explicit path (useful in tests). + /// + /// Hardened after a real incident (see `save`'s test guard): the + /// previous plain `fs::write` was neither atomic nor recoverable. + /// This now (1) keeps one generation of the previous file as + /// `.bak` — best-effort, a failed backup never blocks the save — + /// and (2) writes to a sibling temp file and renames it into place + /// with a bounded retry for transient Windows handle races (AV / + /// search indexer), mirroring `vectordb::store::atomic_write_json`. pub fn save_to(&self, path: &std::path::Path) -> Result<()> { + use std::io::Write; + if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } - fs::write(path, serde_json::to_string_pretty(self)?)?; - Ok(()) + + // (1) One-generation backup, best-effort. + if path.exists() { + let mut bak = path.as_os_str().to_os_string(); + bak.push(".bak"); + let _ = fs::copy(path, std::path::PathBuf::from(bak)); + } + + // (2) Atomic replace: temp file + rename over the target. + let mut tmp = path.as_os_str().to_os_string(); + tmp.push(".new"); + let tmp_path = std::path::PathBuf::from(tmp); + let data = serde_json::to_string_pretty(self)?; + let write_result = (|| -> std::io::Result<()> { + let mut f = fs::File::create(&tmp_path)?; + f.write_all(data.as_bytes())?; + f.sync_all()?; + Ok(()) + })(); + if let Err(e) = write_result { + let _ = fs::remove_file(&tmp_path); + return Err(e.into()); + } + + for attempt in 0..5 { + match fs::rename(&tmp_path, path) { + Ok(()) => return Ok(()), + Err(e) => { + let transient = e + .raw_os_error() + .is_some_and(|raw| matches!(raw, 5 | 32 | 33)) + || { + let msg = e.to_string(); + msg.contains("being used") + || msg.contains("is in use") + || msg.contains("Access is denied") + }; + if transient && attempt < 4 { + std::thread::sleep(std::time::Duration::from_millis(20)); + continue; + } + let _ = fs::remove_file(&tmp_path); + return Err(e.into()); + } + } + } + unreachable!("retry loop always returns") } /// Return the path to the repos config file. diff --git a/src/embed/embedder.rs b/src/embed/embedder.rs index f13883a0..d0c2f606 100644 --- a/src/embed/embedder.rs +++ b/src/embed/embedder.rs @@ -1,11 +1,11 @@ use anyhow::{anyhow, Result}; use fastembed::{EmbeddingModel as FastEmbedModel, InitOptions, TextEmbedding}; -use ort::execution_providers::CPUExecutionProvider; +use ort::ep::CPU; use crate::file::Language; /// Available embedding models -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum ModelType { // === MiniLM Family === /// All-MiniLM-L6-v2 - 384 dimensions, fast and efficient @@ -242,6 +242,24 @@ impl ModelType { } } + /// Resolve the embedding model recorded in an index's `metadata.json`. + /// + /// The reader counterpart to [`Self::write_metadata_fields`]. Returns `None` + /// when the file is missing/unreadable, carries no `model_short_name`, or + /// names a model this build does not know — callers fall back to + /// [`ModelType::default`]. Every query path MUST resolve the model per + /// target index through here (instead of assuming the default): embedding a + /// query with any model other than the one the index was built with either + /// fails with a dimension mismatch or silently compares incomparable vector + /// spaces. Multi-repo serve mode is where this matters most, because one + /// hub can hold indexes built with different models. + pub fn from_index_metadata(db_path: &std::path::Path) -> Option { + let content = std::fs::read_to_string(db_path.join("metadata.json")).ok()?; + let json: serde_json::Value = serde_json::from_str(&content).ok()?; + let name = json.get("model_short_name").and_then(|v| v.as_str())?; + Self::parse(name) + } + pub fn prepare_query(&self, text: &str) -> String { match self { Self::EmbeddingGemma300MQ4 => format!("task: search result | query: {text}"), @@ -312,9 +330,7 @@ impl FastEmbedder { // Use CPU execution provider WITH arena allocator for speed. // Arena allocator provides fast memory reuse during inference. - let cpu_ep = CPUExecutionProvider::default() - .with_arena_allocator(true) - .build(); + let cpu_ep = CPU::default().with_arena_allocator(true).build(); let model = TextEmbedding::try_new( InitOptions::new(model_type.to_fastembed_model()) diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 717ed077..bc0a497f 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -10,6 +10,7 @@ pub use cache::{ pub use embedder::{FastEmbedder, ModelType}; use anyhow::Result; +use std::collections::HashMap; use std::env; use std::sync::{Arc, Mutex}; @@ -298,6 +299,50 @@ impl Default for EmbeddingService { } } +/// Lazily-created, per-model cache of [`EmbeddingService`]s. +/// +/// Serve mode is multi-repo and different repos may be indexed with different +/// embedding models (an older MiniLM index alongside a rebuilt EmbeddingGemma +/// one), so a single shared service is wrong: the query must be embedded with +/// the same model the target index was built with. The pool loads each model at +/// most once per serve instance and reuses it across MCP sessions and REST +/// handlers. Each model gets its own mutex, so queries against different models +/// do not serialise on one global lock. +#[derive(Default)] +pub struct EmbeddingServicePool { + services: Mutex>>>, + cache_dir: Option, +} + +impl EmbeddingServicePool { + /// Create a pool. `cache_dir` overrides the ONNX model cache directory + /// (`None` = fastembed's configured cache, i.e. the global models dir). + pub fn new(cache_dir: Option) -> Self { + Self { + services: Mutex::new(HashMap::new()), + cache_dir, + } + } + + /// Return the service for `model`, loading its ONNX model on first use. + /// + /// The returned `Arc` is locked independently per model, so a caller can + /// hold it across an `embed_query` without blocking other models. + pub fn get(&self, model: ModelType) -> Result>> { + let mut guard = self + .services + .lock() + .map_err(|e| anyhow::anyhow!("Embedding service pool mutex poisoned: {e}"))?; + if let Some(existing) = guard.get(&model) { + return Ok(existing.clone()); + } + let service = EmbeddingService::with_cache_dir(model, self.cache_dir.as_deref())?; + let arc = Arc::new(Mutex::new(service)); + guard.insert(model, arc.clone()); + Ok(arc) + } +} + #[cfg(test)] mod tests { use super::*; @@ -308,6 +353,50 @@ mod tests { assert_eq!(model.dimensions(), 384); } + /// The index-metadata reader must invert `write_metadata_fields`, and must + /// report "no answer" (None) for unknown/missing names rather than silently + /// claiming the default — callers decide the fallback. + #[test] + fn test_model_type_round_trips_through_index_metadata() { + for model in [ + ModelType::AllMiniLML6V2Q, + ModelType::EmbeddingGemma300MQ4, + ModelType::BGEBaseENV15, + ] { + let dir = tempfile::tempdir().unwrap(); + let mut obj = serde_json::Map::new(); + model.write_metadata_fields(&mut obj); + std::fs::write( + dir.path().join("metadata.json"), + serde_json::to_string(&obj).unwrap(), + ) + .unwrap(); + assert_eq!( + ModelType::from_index_metadata(dir.path()), + Some(model), + "reader must invert the writer for '{:?}'", + model + ); + } + + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("metadata.json"), + r#"{"model_short_name":"not-a-real-model"}"#, + ) + .unwrap(); + assert_eq!(ModelType::from_index_metadata(dir.path()), None); + + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("metadata.json"), "{}").unwrap(); + assert_eq!(ModelType::from_index_metadata(dir.path()), None); + + assert_eq!( + ModelType::from_index_metadata(std::path::Path::new("/nonexistent-db-dir")), + None + ); + } + #[test] #[ignore] // Requires model download fn test_embedding_service_creation() { diff --git a/src/federation/mod.rs b/src/federation/mod.rs index 855b2190..d16a56e4 100644 --- a/src/federation/mod.rs +++ b/src/federation/mod.rs @@ -24,6 +24,49 @@ use crate::index::build_serve_client_with_key; // Shared with the `remote` CLI command via constants (single source of truth). use crate::constants::DEFAULT_REMOTE_TIMEOUT_SECS as DEFAULT_TIMEOUT_SECS; +/// Whether an HTTP status from a remote peer is transient — likely a +/// scale-to-zero cold start (503) or a short gateway hiccup (502/504) — and +/// therefore worth a bounded in-call retry. Everything else (4xx auth/config +/// problems, the peer's own 5xx tool errors) is a real answer, not noise: +/// retrying it would only burn the caller's time before failing the same way. +fn is_transient_peer_status(status: reqwest::StatusCode) -> bool { + matches!(status.as_u16(), 502..=504) +} + +/// Backoff before federation retry attempt `retry_idx` (0-based). Honours the +/// `CODESEARCH_REMOTE_RETRY_BACKOFF_MS` test override (a single ms value used +/// for every retry); without it, [`REMOTE_PEER_RETRY_BACKOFF_MS`] supplies one +/// delay per retry, reusing its last entry for any excess retries. +async fn peer_retry_backoff(retry_idx: usize) { + let override_ms = std::env::var(crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV) + .ok() + .and_then(|v| v.parse::().ok()); + let default_ms = || { + crate::constants::REMOTE_PEER_RETRY_BACKOFF_MS + .last() + .copied() + .unwrap_or(8000) + }; + let ms = override_ms.unwrap_or_else(|| { + crate::constants::REMOTE_PEER_RETRY_BACKOFF_MS + .get(retry_idx) + .copied() + .unwrap_or_else(default_ms) + }); + tokio::time::sleep(std::time::Duration::from_millis(ms)).await; +} + +/// Final-failure message for a peer that still answered transiently after all +/// retries: names the likely cause (cold start) and the remedy (retry soon), +/// so a caller can tell "temporarily unavailable" apart from "misconfigured / +/// auth failed" — which need entirely different responses. +fn transient_exhausted_message(what: &str, status: u16, retries: u32) -> String { + format!( + "remote {what} did not respond in time (http={status} after {retries} retries) — \ + likely a cold start on a scale-to-zero host; retry the same call in ~30s" + ) +} + /// A single hit returned by a remote `/search` endpoint. /// /// Fields mirror the local search-item shapes (semantic *and* literal) but are @@ -141,7 +184,7 @@ pub struct RemoteRepoStatus { pub tool_call_count: Option, } -/// `GET /repos/:alias/info` payload — on-disk index stats for one repo on the +/// `GET /repos/{alias}/info` payload — on-disk index stats for one repo on the /// peer. Only the fields the TUI mount-info overlay renders are typed; every /// field is optional/defaulted so an older/newer remote still parses. #[derive(Debug, Clone, Default, Deserialize, Serialize)] @@ -173,7 +216,7 @@ pub struct RemoteRepoAdded { pub message: Option, } -/// `DELETE /repos/:alias` success payload (HTTP 200). +/// `DELETE /repos/{alias}` success payload (HTTP 200). #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct RemoteRepoRemoved { #[serde(default)] @@ -186,7 +229,7 @@ pub struct RemoteRepoRemoved { pub message: Option, } -/// `POST /repos/:alias/reindex` success payload (HTTP 202). +/// `POST /repos/{alias}/reindex` success payload (HTTP 202). #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct RemoteReindexResult { #[serde(default)] @@ -268,43 +311,78 @@ impl FederationClient { /// Shared POST + parse for `/search` (group- and project-scoped variants /// prepare the body differently, then funnel through here). + /// + /// Transient statuses (502/503/504 — most often a scale-to-zero cold + /// start) are retried a bounded number of times inside this call before + /// the failure is surfaced; see [`REMOTE_PEER_RETRY_ATTEMPTS`]. Transport + /// errors (refused/timeout) are NOT retried: the per-request timeout + /// already consumed the call's latency budget, and a hard-unreachable peer + /// fails the same way on the next attempt. async fn post_search( &self, peer: &RemotePeer, body: serde_json::Value, ) -> Outcome> { let url = Self::peer_url(peer, crate::constants::SEARCH_PATH); - let req = self - .client - .post(&url) - .timeout(Self::peer_timeout(peer)) - .json(&body); - let req = attach_bearer(req, &peer.api_key); - - match req.send().await { - Ok(resp) => { - let status = resp.status(); - match resp.json::().await { - Ok(parsed) if status.is_success() && !parsed._mcp_is_error.unwrap_or(false) => { - Outcome::Ok(parsed.results) + let attempts = crate::constants::REMOTE_PEER_RETRY_ATTEMPTS.max(1); + for attempt in 0..attempts { + if attempt > 0 { + peer_retry_backoff((attempt - 1) as usize).await; + } + let req = self + .client + .post(&url) + .timeout(Self::peer_timeout(peer)) + .json(&body); + let req = attach_bearer(req, &peer.api_key); + + match req.send().await { + Ok(resp) => { + let status = resp.status(); + if is_transient_peer_status(status) { + let retries_left = attempts - attempt - 1; + if retries_left > 0 { + tracing::debug!( + "remote /search transient {} (attempt {}/{}) — retrying", + status, + attempt + 1, + attempts + ); + continue; + } + return Outcome::Unreachable(transient_exhausted_message( + "/search", + status.as_u16(), + attempts - 1, + )); } - Ok(parsed) => { - // Tool-level error on the remote (e.g. scope_required). - let n = parsed.results.len(); - Outcome::Unreachable(format!( - "remote /search returned a tool error (http={status}, items={n})" - )) + match resp.json::().await { + Ok(parsed) + if status.is_success() && !parsed._mcp_is_error.unwrap_or(false) => + { + return Outcome::Ok(parsed.results) + } + Ok(parsed) => { + // Tool-level error on the remote (e.g. scope_required). + let n = parsed.results.len(); + return Outcome::Unreachable(format!( + "remote /search returned a tool error (http={status}, items={n})" + )); + } + Err(e) => { + return Outcome::Unreachable(format!( + "remote /search returned non-JSON body (http={status}): {e}" + )) + } } - Err(e) => Outcome::Unreachable(format!( - "remote /search returned non-JSON body (http={status}): {e}" - )), } + Err(e) => return Outcome::Unreachable(format!("remote /search unreachable: {e}")), } - Err(e) => Outcome::Unreachable(format!("remote /search unreachable: {e}")), } + unreachable!("retry loop always returns on its final iteration") } - /// Fetch a single chunk from a remote peer's `/chunk/:id` endpoint. + /// Fetch a single chunk from a remote peer's `/chunk/{id}` endpoint. /// /// Scoping mirrors [`Self::search_project`]: /// - When `remote_alias` is `Some`, the lookup is scoped to that single @@ -324,7 +402,7 @@ impl FederationClient { ) -> Outcome { let mut url = Self::peer_url( peer, - &crate::constants::CHUNK_PATH.replace(":id", &chunk_id.to_string()), + &crate::constants::CHUNK_PATH.replace("{id}", &chunk_id.to_string()), ); // Scope the lookup: prefer a single-project scope (`project=`) // so the multi-repo peer can disambiguate the chunk_id; fall back to @@ -349,28 +427,56 @@ impl FederationClient { .join("&"); url.push('?'); url.push_str(&query); - let req = self.client.get(&url).timeout(Self::peer_timeout(peer)); - let req = attach_bearer(req, &peer.api_key); - match req.send().await { - Ok(resp) => { - let status = resp.status(); - match resp.json::().await { - Ok(v) if status.is_success() && !is_mcp_error(&v) => Outcome::Ok(v), - Ok(v) => Outcome::Unreachable(format!( - "remote /chunk returned a tool error (http={status}): {}", - short_reason(&v) - )), - Err(e) => Outcome::Unreachable(format!( - "remote /chunk returned non-JSON body (http={status}): {e}" - )), + let attempts = crate::constants::REMOTE_PEER_RETRY_ATTEMPTS.max(1); + for attempt in 0..attempts { + if attempt > 0 { + peer_retry_backoff((attempt - 1) as usize).await; + } + let req = self.client.get(&url).timeout(Self::peer_timeout(peer)); + let req = attach_bearer(req, &peer.api_key); + match req.send().await { + Ok(resp) => { + let status = resp.status(); + if is_transient_peer_status(status) { + let retries_left = attempts - attempt - 1; + if retries_left > 0 { + tracing::debug!( + "remote /chunk transient {} (attempt {}/{}) — retrying", + status, + attempt + 1, + attempts + ); + continue; + } + return Outcome::Unreachable(transient_exhausted_message( + "/chunk", + status.as_u16(), + attempts - 1, + )); + } + match resp.json::().await { + Ok(v) if status.is_success() && !is_mcp_error(&v) => return Outcome::Ok(v), + Ok(v) => { + return Outcome::Unreachable(format!( + "remote /chunk returned a tool error (http={status}): {}", + short_reason(&v) + )) + } + Err(e) => { + return Outcome::Unreachable(format!( + "remote /chunk returned non-JSON body (http={status}): {e}" + )) + } + } } + Err(e) => return Outcome::Unreachable(format!("remote /chunk unreachable: {e}")), } - Err(e) => Outcome::Unreachable(format!("remote /chunk unreachable: {e}")), } + unreachable!("retry loop always returns on its final iteration") } /// Shared request/response handling for the management endpoints - /// (`/status`, `/repos`, `/repos/:alias`, `/repos/:alias/reindex`). + /// (`/status`, `/repos`, `/repos/{alias}`, `/repos/{alias}/reindex`). /// /// Distinguishes three failure modes (see [`ManagementOutcome`]): /// transport failure → `Unreachable`; non-2xx → `HttpError` with the peer's @@ -471,7 +577,7 @@ impl FederationClient { .await } - /// `DELETE /repos/:alias` — unregister a repo on the peer and delete its DB. + /// `DELETE /repos/{alias}` — unregister a repo on the peer and delete its DB. /// `alias` is the peer's repo alias (NOT a local path). pub async fn remove_repo( &self, @@ -485,7 +591,7 @@ impl FederationClient { .await } - /// `GET /repos/:alias/info` — fetch on-disk index stats (chunks/files/db + /// `GET /repos/{alias}/info` — fetch on-disk index stats (chunks/files/db /// size/model) for one repo on the peer. `alias` is the peer's repo alias. pub async fn repo_info( &self, @@ -502,7 +608,7 @@ impl FederationClient { .await } - /// `POST /repos/:alias/reindex[?force=true]` — trigger a background + /// `POST /repos/{alias}/reindex[?force=true]` — trigger a background /// incremental (or forced full) reindex of a repo on the peer. pub async fn reindex( &self, @@ -760,7 +866,7 @@ mod tests { // `group`) for a namespaced lookup — the fix for `ambiguous_chunk_id` // on a multi-repo peer. let app = axum::Router::new().route( - "/chunk/:id", + "/chunk/{id}", axum::routing::get( |axum::extract::Query(params): axum::extract::Query< std::collections::HashMap, @@ -812,7 +918,7 @@ mod tests { // lookup must then fall back to the peer's group scope and NOT send a // `project` param — preserving pre-fix behaviour for old refs. let app = axum::Router::new().route( - "/chunk/:id", + "/chunk/{id}", axum::routing::get( |axum::extract::Query(params): axum::extract::Query< std::collections::HashMap, @@ -935,7 +1041,7 @@ mod tests { async fn remove_repo_targets_alias_in_url() { // Echo the captured alias back to prove it landed in the DELETE path. let app = axum::Router::new().route( - "/repos/:alias", + "/repos/{alias}", axum::routing::delete( |axum::extract::Path(alias): axum::extract::Path| async move { axum::Json(serde_json::json!({ @@ -966,7 +1072,7 @@ mod tests { async fn reindex_posts_to_alias_reindex_path() { // Capture the alias from the path to prove the reindex URL was built. let app = axum::Router::new().route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post( |axum::extract::Path(alias): axum::extract::Path| async move { axum::Json(serde_json::json!({ @@ -1000,7 +1106,7 @@ mod tests { async fn reindex_with_force_appends_force_query() { // Capture the query string to prove ?force=true was forwarded. let app = axum::Router::new().route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post( |axum::extract::Query(params): axum::extract::Query< std::collections::HashMap, @@ -1034,7 +1140,7 @@ mod tests { // An alias with a space must be percent-encoded on the wire and decoded // back by axum — proves the encoding round-trips through the HTTP layer. let app = axum::Router::new().route( - "/repos/:alias", + "/repos/{alias}", axum::routing::delete( |axum::extract::Path(alias): axum::extract::Path| async move { axum::Json(serde_json::json!({ @@ -1108,4 +1214,246 @@ mod tests { other => panic!("expected Unreachable, got {:?}", other), } } + + // ========================================================================= + // Transient-status retry (502/503/504 — scale-to-zero cold starts, todo #58) + // ========================================================================= + + /// Helper: an axum route answering `/chunk/{id}` that returns 503 (with a + /// non-JSON body, like a real cold-starting gateway) for the first + /// `fail_first` calls, then a valid 200 JSON chunk payload. Counts every + /// hit so tests can assert exactly how many attempts were made. + fn flaky_chunk_route( + fail_first: u32, + ) -> (axum::Router, std::sync::Arc) { + use axum::response::IntoResponse; + let hits = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let hits_clone = hits.clone(); + let router = axum::Router::new().route( + "/chunk/{id}", + axum::routing::get(move || { + let hits = hits_clone.clone(); + async move { + let n = hits.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + if n <= fail_first { + ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "backend cold start", + ) + .into_response() + } else { + axum::Json(serde_json::json!({ + "chunk_id": n, + "content": "warm", + "path": "kb/warm.md", + "start_line": 1, + "end_line": 2 + })) + .into_response() + } + } + }), + ); + (router, hits) + } + + #[tokio::test] + #[serial_test::serial] + async fn get_chunk_retries_transient_503_and_succeeds() { + let _env = crate::testing::EnvRestore::set(&[( + crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV, + "1", + )]); + // Fails twice with 503 (non-JSON body, exactly like the observed + // cold-start failure), then succeeds — must surface as Ok with the + // caller never seeing the transient failures. + let (router, hits) = flaky_chunk_route(2); + let addr = spawn_test_server(router).await; + + let client = FederationClient::new().unwrap(); + let outcome = client + .get_chunk(&peer(format!("http://{addr}")), Some("kb"), 42, None) + .await; + match outcome { + Outcome::Ok(v) => assert_eq!(v["content"], "warm"), + other => panic!("expected Ok after retry, got {other:?}"), + } + assert_eq!( + hits.load(std::sync::atomic::Ordering::SeqCst), + 3, + "two cold-start 503s + one success = three attempts" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn get_chunk_requests_the_exact_chunk_path_without_placeholder_residue() { + // Regression (todo #153): the URL template is `/chunk/{id}`; a + // placeholder replacement that missed the closing brace produced + // `/chunk/2058%7D`. Axum's `{id}` param happily swallowed the stray + // `}` into the captured value, so mock-based route tests passed while + // real peers answered 400 Bad Request on the mangled id. The route + // below echoes back the EXACT path it was hit on, so any residue in + // the constructed URL fails the assertion. + let seen = std::sync::Arc::new(tokio::sync::Mutex::new(String::new())); + let seen_clone = seen.clone(); + let router = axum::Router::new().route( + "/chunk/{id}", + axum::routing::get(move |uri: axum::http::Uri| { + let seen = seen_clone.clone(); + async move { + *seen.lock().await = uri.path().to_string(); + axum::Json(serde_json::json!({ + "chunk_id": 2058, + "content": "ok", + "path": "kb/x.md", + "start_line": 1, + "end_line": 2 + })) + } + }), + ); + let addr = spawn_test_server(router).await; + + let client = FederationClient::new().unwrap(); + let outcome = client + .get_chunk(&peer(format!("http://{addr}")), Some("bynder"), 2058, None) + .await; + assert!( + matches!(outcome, Outcome::Ok(_)), + "the peer must answer the clean path" + ); + assert_eq!( + *seen.lock().await, + "/chunk/2058", + "the constructed URL must carry the bare chunk id — no placeholder residue" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn get_chunk_persistent_503_reports_cold_start_hint() { + let _env = crate::testing::EnvRestore::set(&[( + crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV, + "1", + )]); + // Always 503: exhaust the retries, then fail with a message that + // names the likely cause and the remedy instead of a raw "non-JSON + // body (http=503)" the caller can do nothing with. + let (router, hits) = flaky_chunk_route(u32::MAX); + let addr = spawn_test_server(router).await; + + let client = FederationClient::new().unwrap(); + let outcome = client + .get_chunk(&peer(format!("http://{addr}")), Some("kb"), 7, None) + .await; + match outcome { + Outcome::Unreachable(msg) => { + assert!(msg.contains("503"), "message should name the status: {msg}"); + assert!( + msg.contains("cold start"), + "message should name the likely cause: {msg}" + ); + assert!( + msg.contains("~30s"), + "message should tell the caller when to retry: {msg}" + ); + assert!( + msg.contains("after 2 retries"), + "message should state the retry count: {msg}" + ); + } + other => panic!("expected Unreachable, got {other:?}"), + } + assert_eq!( + hits.load(std::sync::atomic::Ordering::SeqCst), + crate::constants::REMOTE_PEER_RETRY_ATTEMPTS, + "initial attempt + configured retries, no more" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn get_chunk_does_not_retry_non_transient_status() { + let _env = crate::testing::EnvRestore::set(&[( + crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV, + "1", + )]); + // A 500 with a JSON error body is the peer's OWN answer (the cloud + // peer rejects force-reindex this way) — retrying it would only burn + // time before failing identically. Exactly one request must be made. + let hits = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let hits_clone = hits.clone(); + let router = axum::Router::new().route( + "/chunk/{id}", + axum::routing::get(move || { + let hits = hits_clone.clone(); + async move { + hits.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + axum::http::StatusCode::INTERNAL_SERVER_ERROR + } + }), + ); + let addr = spawn_test_server(router).await; + + let client = FederationClient::new().unwrap(); + let outcome = client + .get_chunk(&peer(format!("http://{addr}")), Some("kb"), 1, None) + .await; + assert!( + matches!(outcome, Outcome::Unreachable(_)), + "500 must fail, not retry" + ); + assert_eq!( + hits.load(std::sync::atomic::Ordering::SeqCst), + 1, + "non-transient status must not be retried" + ); + } + + #[tokio::test] + #[serial_test::serial] + async fn search_retries_transient_503_and_succeeds() { + let _env = crate::testing::EnvRestore::set(&[( + crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV, + "1", + )]); + // Same exposure as get_chunk (identical pre-fix code shape) — one 503 + // then a valid 200 search response must surface as Ok. + let hits = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let hits_clone = hits.clone(); + let router = axum::Router::new().route( + crate::constants::SEARCH_PATH, + axum::routing::post(move || { + use axum::response::IntoResponse; + let hits = hits_clone.clone(); + async move { + let n = hits.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + if n == 1 { + return (axum::http::StatusCode::SERVICE_UNAVAILABLE, "cold start") + .into_response(); + } + axum::Json(serde_json::json!({ + "results": [{ "path": "kb/doc.md", "score": 0.9 }] + })) + .into_response() + } + }), + ); + let addr = spawn_test_server(router).await; + + let client = FederationClient::new().unwrap(); + let outcome = client + .search_project( + &peer(format!("http://{addr}")), + serde_json::json!({"query": "x"}), + "kb", + ) + .await; + match outcome { + Outcome::Ok(items) => assert_eq!(items.len(), 1), + other => panic!("expected Ok after retry, got {other:?}"), + } + assert_eq!(hits.load(std::sync::atomic::Ordering::SeqCst), 2); + } } diff --git a/src/fts/tantivy_store.rs b/src/fts/tantivy_store.rs index 459980f5..45e7eba4 100644 --- a/src/fts/tantivy_store.rs +++ b/src/fts/tantivy_store.rs @@ -177,16 +177,17 @@ impl FtsStore { /// Open or create index with retry logic for Windows file locking issues fn open_or_create_index_with_retry(fts_path: &Path, schema: &Schema) -> Result { - let max_retries = 3; + const MAX_RETRIES: usize = 3; let mut last_error: Option = None; - for attempt in 0..max_retries { + for attempt in 0..MAX_RETRIES { if attempt > 0 { // Wait before retry (exponential backoff) std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt))); } - let result: Result = if fts_path.join("meta.json").exists() { + let opened_existing = fts_path.join("meta.json").exists(); + let result: Result = if opened_existing { Index::open_in_dir(fts_path).map_err(|e| e.to_string()) } else { MmapDirectory::open(fts_path) @@ -202,18 +203,53 @@ impl FtsStore { Err(e) => { last_error = Some(e); // On Windows, try to clear lock files if permission denied - if attempt < max_retries - 1 { + if attempt < MAX_RETRIES - 1 { Self::try_clear_lock_files(fts_path); } } } } - Err(anyhow!( - "Failed to open FTS index after {} retries: {}", - max_retries, - last_error.unwrap_or_default() - )) + // Last resort: a pre-upgrade FTS index written by an older tantivy + // major cannot be opened by this one (format break), which would + // brick the whole DB for existing users. The FTS index is derived + // data — fully rebuildable via reindex — so wipe it and create a + // fresh empty index instead of failing. BM25 results stay empty + // until the next (re)index; vector search is unaffected. + if let Err(wipe_err) = Self::wipe_fts_dir(fts_path) { + return Err(anyhow!( + "Failed to open FTS index after {MAX_RETRIES} retries: {} (wipe also failed: {wipe_err})", + last_error.unwrap_or_default() + )); + } + tracing::warn!( + "FTS index at {} was unreadable by this codesearch version and has been reset; \ + run 'codesearch index' to rebuild it", + fts_path.display() + ); + MmapDirectory::open(fts_path) + .map_err(|e| e.to_string()) + .and_then(|dir| { + Index::create(dir, schema.clone(), IndexSettings::default()) + .map_err(|e| e.to_string()) + }) + .map_err(|e| { + anyhow!( + "Failed to open FTS index after {} retries: {} (fresh create also failed: {})", + MAX_RETRIES, + last_error.unwrap_or_default(), + e + ) + }) + } + + /// Remove the FTS index directory contents so a fresh index can be created. + fn wipe_fts_dir(fts_path: &Path) -> std::io::Result<()> { + if !fts_path.exists() { + return Ok(()); + } + std::fs::remove_dir_all(fts_path)?; + std::fs::create_dir_all(fts_path) } /// Create writer with retry logic for Windows file locking issues @@ -502,7 +538,8 @@ impl FtsStore { }; // Execute search - let top_docs = searcher.search(&parsed_query, &TopDocs::with_limit(limit))?; + let top_docs = + searcher.search(&parsed_query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -560,7 +597,7 @@ impl FtsStore { BooleanQuery::union(vec![Box::new(boosted_sig), Box::new(content_query)]) }; - let top_docs = searcher.search(&combined, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&combined, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -578,7 +615,7 @@ impl FtsStore { let query = RegexQuery::from_pattern(pattern, self.content_field) .map_err(|e| anyhow!("Invalid regex pattern '{}': {}", pattern, e))?; - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -628,7 +665,7 @@ impl FtsStore { Box::new(PhraseQuery::new(terms)) }; - let top_docs = searcher.search(&query, &TopDocs::with_limit(limit))?; + let top_docs = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; self.collect_fts_results(top_docs) } @@ -845,4 +882,32 @@ mod tests { Ok(()) } + + /// An index written by an incompatible tantivy major must not brick the + /// store: the unreadable index is wiped and a fresh empty one created + /// (BM25 rebuilds on the next index run). + #[test] + fn unreadable_index_is_wiped_and_store_recovers() -> Result<()> { + let tmp = tempfile::tempdir()?; + let fts_dir = tmp.path().join("fts"); + std::fs::create_dir_all(&fts_dir)?; + // A meta.json no tantivy version can parse simulates an index from + // an incompatible tantivy major. + std::fs::write(fts_dir.join("meta.json"), "{ not valid tantivy metadata")?; + + let mut store = FtsStore::new(tmp.path())?; + + // Fresh store: opens and searches empty. + let results = store.search("anything", 10, None)?; + assert!(results.is_empty()); + + // And accepts writes after the wipe. + store.add_chunk(1, "recovery probe content", "probe.rs", None, "block")?; + store.commit()?; + let results = store.search("recovery probe", 10, None)?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].chunk_id, 1); + + Ok(()) + } } diff --git a/src/index/manager.rs b/src/index/manager.rs index a0f0bc01..c0d54bc4 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -504,7 +504,7 @@ impl IndexManager { /// /// Fails fast if metadata is missing, names an unknown model, or records a /// dimension count that disagrees with the resolved model. - fn resolve_embed_model(db_path: &Path) -> Result<(ModelType, usize)> { + pub(crate) fn resolve_embed_model(db_path: &Path) -> Result<(ModelType, usize)> { let metadata_path = db_path.join("metadata.json"); if !metadata_path.exists() { return Err(anyhow::anyhow!( @@ -2489,6 +2489,35 @@ mod tests { use crate::cache::FileMetaStore; use tempfile::tempdir; + /// Dropping the last `Arc` must release every handle inside + /// the DB directory — the precondition `index rm`'s delete path depends on. + /// + /// Pins the Windows manifestation of a heed 0.20 leak fixed in + /// `TrackedEnv::drop`: the `OPENED_ENV` cache entry held a strong `Env` + /// clone, so `mdb_env_close` never ran after a plain drop and + /// `data.mdb`/`lock.mdb` stayed locked for the life of the process — + /// `serve::tests::index_rm_deletes_db_while_serve_holds_real_lmdb_env` + /// failed deterministically on os error 32 through the whole 60 s retry + /// budget with an EMPTY LMDB registry (the holder was invisible to it). + /// This is the serve-free, instant version of that acceptance test. + #[test] + fn sharedstores_drop_releases_db_dir_for_deletion() { + let tmp = tempdir().unwrap(); + let db = tmp.path().join(".codesearch.db"); + let stores = SharedStores::new(&db, 384).expect("open SharedStores"); + assert!( + !crate::lmdb_registry::open_holders_under(&db).is_empty(), + "holder must be visible while SharedStores lives" + ); + drop(stores); + assert!( + crate::lmdb_registry::open_holders_under(&db).is_empty(), + "registry must drain after the last Arc drops" + ); + std::fs::remove_dir_all(&db) + .expect("db dir must be deletable after SharedStores drops (no leaked LMDB handles)"); + } + /// Helper: create metadata.json in db_path with given dimensions fn create_metadata_json(db_path: &Path, dimensions: usize) { let metadata = serde_json::json!({ diff --git a/src/index/mod.rs b/src/index/mod.rs index 8637fdb6..273c460a 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -9,9 +9,7 @@ use tracing::{debug, info}; use crate::cache::{normalize_path, safe_canonicalize, FileMetaStore}; use crate::chunker::SemanticChunker; -use crate::db_discovery::{ - find_best_database, is_registered_repository, register_repository, unregister_repository, -}; +use crate::db_discovery::{find_best_database, is_registered_repository, register_repository}; use crate::embed::{EmbeddingService, ModelType}; use crate::file::FileWalker; use crate::fts::FtsStore; @@ -49,6 +47,58 @@ pub(crate) fn ensure_hnsw_index_if_needed( } } +/// Dimensions recorded in an index's `metadata.json` (fallback: the default). +/// +/// The vector store MUST be opened with the dimensions the index was built +/// with. `codesearch stats` / `get_db_stats` used to pass a hardcoded 384, so +/// they reported `Dimensions: 384` for every index — including 768-dim +/// EmbeddingGemma ones — and opened those stores with the wrong dimension. +fn recorded_dimensions(db_path: &Path) -> usize { + let from_metadata = std::fs::read_to_string(db_path.join("metadata.json")) + .ok() + .and_then(|c| serde_json::from_str::(&c).ok()) + .and_then(|j| j.get("dimensions").and_then(|v| v.as_u64())) + .map(|d| d as usize); + from_metadata + .or_else(|| ModelType::from_index_metadata(db_path).map(|m| m.dimensions())) + .unwrap_or(crate::constants::DEFAULT_EMBEDDING_DIMENSIONS) +} + +/// Resolve the embedding model for an indexing run. +/// +/// On an existing index the model recorded in `metadata.json` wins: embedding +/// with any other model (including the hardcoded default) either fails with a +/// dimension mismatch or silently mixes vector spaces. An explicit `--model` +/// that disagrees with the recorded model is rejected — the index must be +/// rebuilt with `--force` to change models. +/// +/// This mirrors `IndexManager::resolve_embed_model`, which the serve and +/// watcher paths already use; the CLI `index` path used `model.unwrap_or_default()` +/// and therefore downgraded any non-default index to 384-dim MiniLM. +fn resolve_index_model( + db_path: &Path, + force: bool, + requested: Option, +) -> Result { + // `--force` deletes the database (see `get_db_path_smart`), so there is no + // recorded model to honour; a non-existent/legacy index has none either. + if force || !db_path.join("metadata.json").exists() { + return Ok(requested.unwrap_or_default()); + } + let (recorded, _dims) = IndexManager::resolve_embed_model(db_path)?; + match requested { + Some(req) if req != recorded => Err(anyhow::anyhow!( + "Model mismatch: {} was indexed with '{}', but --model '{}' was requested.\n\ + To change models, rebuild the index: codesearch --model {} index --force", + db_path.display(), + recorded.short_name(), + req.short_name(), + req.short_name() + )), + _ => Ok(recorded), + } +} + /// Update metadata.json with current chunk/file counts so that `status(projects)` /// can report accurate numbers without opening LMDB. /// Uses atomic read-modify-write (temp+rename) so a crash never leaves an empty file. @@ -551,7 +601,11 @@ async fn index_with_options( cancel_token: CancellationToken, ) -> Result<()> { let (db_path, project_path) = get_db_path_smart(path, global, force)?; - let model_type = model.unwrap_or_default(); + // Resolve the embedding model BEFORE touching the index: on an existing + // index the model recorded in metadata.json wins, and an explicit `--model` + // that disagrees is rejected (rebuild with --force). Defaulting here would + // embed 384-dim MiniLM vectors into a 768-dim index. See `resolve_index_model`. + let model_type = resolve_index_model(&db_path, force, model)?; // Macro to conditionally print macro_rules! log_print { @@ -716,7 +770,7 @@ async fn index_with_options( if total_chunks_to_delete > 0 { log_print!("\n🔄 Deleting {} old chunks...", total_chunks_to_delete); - let mut store = VectorStore::new(&db_path, 384)?; // Will load dimensions from DB + let mut store = VectorStore::new(&db_path, model_type.dimensions())?; let mut fts_store = FtsStore::new_with_writer(&db_path)?; // Delete deleted files' metadata and chunks @@ -1294,7 +1348,7 @@ pub async fn stats(path: Option) -> Result<()> { println!("💾 Database: {}", db_path.display()); println!("📂 Project: {}", project_path.display()); - let store = VectorStore::new(&db_path, 384)?; // We'll need to store dimensions in metadata + let store = VectorStore::new(&db_path, recorded_dimensions(&db_path))?; let stats = store.stats()?; println!("\n{}", "Vector Store:".bright_green()); @@ -1369,7 +1423,7 @@ fn print_repo_stats(repo_path: &Path, db_path: &Path) -> Result<()> { println!(" 📂 {}", repo_path.display()); // Try to load stats - match VectorStore::new(db_path, 384) { + match VectorStore::new(db_path, recorded_dimensions(db_path)) { Ok(store) => match store.stats() { Ok(stats) => { println!( @@ -1645,10 +1699,35 @@ pub async fn remove_from_index(path: Option, keep_config: bool) -> Resu // which the serve endpoint doesn't support — serve always unregisters). if !keep_config { match try_delegate_rm_to_serve(&effective_path).await { - Ok((alias, _)) => { + Ok(removed) => { println!("\n{}", "✅ Delegated to running serve instance.".green()); - println!(" Removed alias '{}'.", alias); - println!(" FSW stopped, repo evicted from memory, DB deleted."); + println!(" Removed alias '{}'.", removed.alias); + println!(" FSW stopped, repo evicted from memory, unregistered from repos.json."); + if removed.db_deleted { + println!(" Database files deleted."); + } else { + // Serve answered 200 but its DB dir survived the delete + // (`removed_db_locked`): report what actually happened — + // never claim a delete that did not occur. This stays Ok + // rather than Err because serve has ALREADY unregistered + // the alias: the Layer-1 local-path error text + // ("repos.json was NOT modified") would be false here, and + // the leftover files are retryable — with the alias gone + // from repos.json the next run skips delegation and + // finishes via the local file-delete path below. + let db_dir = removed.project_path.join(crate::constants::DB_DIR_NAME); + eprintln!( + "⚠️ Database files could not be deleted and are still on disk at {}: {}", + db_dir.display(), + removed + .db_delete_error + .as_deref() + .unwrap_or("unknown error") + ); + eprintln!( + "The alias is already unregistered; re-run the same command to retry the file delete." + ); + } return Ok(()); } Err(reason) => { @@ -1669,51 +1748,64 @@ pub async fn remove_from_index(path: Option, keep_config: bool) -> Resu return Ok(()); } - // Auto-unregister from repos.json unless --keep-config + // FILE REMOVAL FIRST. The previous order (unregister from repos.json, + // then delete the files) left an inconsistent state whenever the delete + // failed — typically LMDB files still held by a serve instance the + // delegation probe could not see (a second serve on another port, a + // crashed one, or a CLI process holding the env): the entry was already + // gone from repos.json while the still-locked database sat on disk, and + // nothing could clean that up except a manual serve-stop. Now repos.json + // is only mutated after the files are actually gone, so a failed delete + // leaves the config untouched and the same command can simply be retried. + if has_local { + if has_global { + println!( + "\n{}", + "⚠️ Warning: Both local and global indexes exist!".yellow() + ); + } + println!("\n{}", "Removing local index...".cyan()); + if let Err(e) = fs::remove_dir_all(&local_db) { + eprintln!( + "⚠️ Database files may be locked by a running codesearch serve. \ + repos.json was NOT modified — stop the locking process and re-run \ + the same command." + ); + return Err(anyhow::anyhow!("Failed to remove local index: {}", e)); + } + println!("{}", "✅ Local index removed!".green()); + } + + // Files are gone (or never existed) — only now update repos.json. + // (This also folds the old "both exist" early-return into the same + // files-then-config flow: that path used to print "(Global index + // remains)" AFTER the unregister above had already removed the global + // entry, and the global-only path used to unregister twice.) if !keep_config { let mut config = crate::db_discovery::repos::ReposConfig::load().unwrap_or_default(); if config.unregister_path(&canonical_path) { if let Err(e) = config.save() { - eprintln!("⚠️ Failed to update repos config: {}", e); - } else { - println!("{}", "🗑️ Unregistered from repos.json".green()); + // The files are gone but the registry still names them. Say so + // and fail loudly instead of reporting partial success as Ok — + // re-running the same command finishes the job (the file + // removal step is now a no-op). + eprintln!( + "⚠️ Local index files were removed, but saving repos.json failed. \ + The config entry remains; re-run the same command to unregister it." + ); + return Err(anyhow::anyhow!("Failed to update repos config: {}", e)); } + println!("{}", "🗑️ Unregistered from repos.json".green()); } } else { println!("{}", "ℹ️ Config entry preserved.".cyan()); } - // If both exist (shouldn't happen), remove local with warning - if has_local && has_global { - println!( - "\n{}", - "⚠️ Warning: Both local and global indexes exist!".yellow() - ); - println!(" Removing local index..."); - if let Err(e) = fs::remove_dir_all(&local_db) { - eprintln!( - "⚠️ Database files may be locked by a running codesearch serve. Stop it and retry." - ); - return Err(anyhow::anyhow!("Failed to remove local index: {}", e)); - } - println!(" {}", "✅ Local index removed".green()); - println!(" (Global index remains)"); - return Ok(()); - } - - // Remove whichever exists - if has_local { - println!("\n{}", "Removing local index...".cyan()); - if let Err(e) = fs::remove_dir_all(&local_db) { - eprintln!( - "⚠️ Database files may be locked by a running codesearch serve. Stop it and retry." - ); - return Err(anyhow::anyhow!("Failed to remove local index: {}", e)); - } - println!("{}", "✅ Local index removed!".green()); - } else if has_global { - println!("\n{}", "Removing global index...".cyan()); - unregister_repository(&canonical_path)?; + // `!keep_config` matters: with --keep-config the entry is deliberately + // left in repos.json, so claiming the global index was removed would + // contradict the "Config entry preserved." line printed just above and + // send the caller looking for a cleanup that never happened. + if !has_local && has_global && !keep_config { println!("{}", "✅ Global index removed!".green()); } @@ -1777,7 +1869,7 @@ async fn get_db_stats(db_path: &Path) -> Result { } // Try to get stats from vector store - let store = VectorStore::new(db_path, 384)?; + let store = VectorStore::new(db_path, recorded_dimensions(db_path))?; let stats = store.stats()?; // Calculate database size @@ -1926,8 +2018,8 @@ const SERVE_HEALTH_RETRY_SLEEP: std::time::Duration = std::time::Duration::from_ /// makes. This is required: /// - when serve is bound to a non-localhost address (the `require_auth_for_network` /// middleware guards ALL endpoints, including `/health`), and -/// - for management endpoints (`POST /repos`, `DELETE /repos/:alias`, -/// `POST /repos/:alias/reindex`, `POST /reload`) when serve is bound to +/// - for management endpoints (`POST /repos`, `DELETE /repos/{alias}`, +/// `POST /repos/{alias}/reindex`, `POST /reload`) when serve is bound to /// localhost with the key set. /// /// Without this, delegation to a network-bound serve returns 401 and falls back @@ -2349,13 +2441,29 @@ pub(crate) async fn try_delegate_add_to_serve( } } +/// The outcome a running serve instance reported for a delegated `index rm` +/// — the parsed `DELETE /repos/{alias}` success payload. +/// +/// `db_deleted == false` means the repo is functionally removed (FSW stopped, +/// evicted from memory, unregistered from repos.json) but the database +/// directory is still on disk: serve's lock-class retry budget was exhausted +/// by a transient `Arc` holder. Callers must report that +/// honestly instead of claiming the files were deleted (BUG2 class). +pub(crate) struct ServeRemoval { + pub(crate) alias: String, + pub(crate) project_path: PathBuf, + pub(crate) db_deleted: bool, + pub(crate) db_delete_error: Option, +} + /// Try to delegate `index rm` to a running serve instance. /// -/// Returns `Ok((alias, project_path))` if the serve accepted the remove request. +/// Returns `Ok(removed)` — including serve's honest DB-delete outcome — if +/// the serve accepted the remove request. /// Returns `Err(reason)` with a human-readable reason if delegation failed. pub(crate) async fn try_delegate_rm_to_serve( path: &Option, -) -> std::result::Result<(String, PathBuf), String> { +) -> std::result::Result { use crate::constants::{resolve_serve_host, DEFAULT_SERVE_PORT, SERVE_PORT_ENV}; let port: u16 = std::env::var(SERVE_PORT_ENV) @@ -2420,15 +2528,58 @@ pub(crate) async fn try_delegate_rm_to_serve( .map(|(a, _)| a.clone()) .ok_or_else(|| format!("path '{}' not found in repos.json", project_path.display()))?; - // 3. DELETE /repos/:alias - let delete_resp = client + // 3. DELETE /repos/{alias} + // + // The DELETE needs its OWN client with a timeout that covers serve's + // legitimate worst-case removal time: `remove_repo` can spend up to + // BG_TASK_COOPERATIVE_TIMEOUT_SECS per cooperative join (FSW + index + // task) plus the full DB_DELETE_RETRY_BUDGET_SECS lock-class retry + // window while transient holders release. Reusing the 3 s health-probe + // client here fired the CLI's own timeout MID-REMOVAL, surfaced as + // "delete failed: operation timed out", and fell through to the local + // path — whose delete then failed on the files serve was still tearing + // down. That made "stop serve and re-run" the only working flow, which + // is exactly what todo #48 Layer 2 removes. The health probe KEEPS the + // short timeout: it must classify Down/Unresponsive quickly. + let delete_client = build_serve_client(std::time::Duration::from_secs( + crate::constants::DB_DELETE_RETRY_BUDGET_SECS + + 2 * crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS + + crate::constants::RM_DELEGATE_DELETE_MARGIN_SECS, + ))?; + let delete_resp = delete_client .delete(format!("{}/repos/{}", base_url, alias)) .send() .await .map_err(|e| format!("delete failed: {}", e))?; if delete_resp.status().is_success() { - Ok((alias, project_path)) + // BUG2-class honesty: a 200 from `remove_repo_handler` is NOT a + // guarantee the DB files are gone — serve reports the real outcome in + // the body (`db_deleted` / `db_delete_error`) because a transient + // Arc holder can outlast its lock-class retry budget. + // Parse and carry it; returning only `(alias, path)` flattened that + // to plain success and the CLI printed "DB deleted." for files that + // were still on disk. + let body = delete_resp.text().await.unwrap_or_default(); + let payload: serde_json::Value = + serde_json::from_str(&body).unwrap_or(serde_json::Value::Null); + // Pre-BUG2 serves have no `db_deleted` field and only ever answered + // "removed" — defaulting to true keeps their behavior unchanged + // instead of fabricating a failure they did not report. + let db_deleted = payload + .get("db_deleted") + .and_then(|v| v.as_bool()) + .unwrap_or(true); + let db_delete_error = payload + .get("db_delete_error") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + Ok(ServeRemoval { + alias, + project_path, + db_deleted, + db_delete_error, + }) } else { let status = delete_resp.status(); let text = delete_resp.text().await.unwrap_or_default(); @@ -2760,3 +2911,532 @@ mod index_quality_tests { assert!(!rebuilt, "empty DB needs no rebuild"); } } + +/// Regression tests for `remove_from_index`'s operation ORDER (todo #48). +/// +/// The bug: repos.json was unregistered BEFORE the database files were +/// deleted. When the delete failed — typically LMDB files locked by a serve +/// instance the delegation probe could not see — the command errored out +/// with the config entry already gone: the registry claimed the repo no +/// longer existed while its still-locked database sat on disk. The fix +/// deletes the files first and only mutates repos.json on success, so a +/// failed delete leaves the config untouched and the same command can be +/// retried after stopping the locking process. +#[cfg(test)] +mod remove_order_tests { + use super::remove_from_index; + use crate::db_discovery::repos::ReposConfig; + use crate::testing::EnvRestore; + use serial_test::serial; + use std::path::{Path, PathBuf}; + + // Every test in this module is `#[serial]`: they mutate the + // process-global CODESEARCH_REPOS_CONFIG (the same var the doctor + // tests read) and CODESEARCH_SERVE_PORT. The EnvRestore guard returned + // by seed_repos_config snapshots and restores both on drop, so even a + // panicking assertion cannot leak a stale value into a later test. + + /// Create a project dir and return its CANONICAL path. Everything in + /// these tests (db path, registered path, assertions) must use the + /// canonical form — `ReposConfig::register` canonicalizes before + /// storing, and on Windows CI the temp root sits under an 8.3 short + /// name (`RUNNER~1`) that only `canonicalize` resolves (the same trap + /// the MSYS regression tests hit in PR #197). + fn make_proj(tmp: &std::path::Path, name: &str) -> PathBuf { + let raw = tmp.join(name); + std::fs::create_dir(&raw).unwrap(); + crate::cache::safe_canonicalize(&raw).unwrap() + } + + /// Bind an ephemeral listener whose connections are accepted and closed + /// immediately (TCP RST). Pointing CODESEARCH_SERVE_PORT here makes the + /// delegation health probe inside `remove_from_index` fail FAST and + /// hermetically: the reset surfaces as a non-timeout connect error, which + /// the probe classifies as `ServeProbe::Down` on the first attempt. + /// Without this the probe would either reach a REAL serve on the default + /// port (live DELETE /repos/ against the developer's registry — + /// observed during review) or, on an unused port, hit the Windows + /// loopback hang: 3 retries x 3s client timeout of dead air per + /// delegating test. + async fn spawn_reset_server() -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + loop { + if listener.accept().await.is_err() { + break; + } + // The accepted TcpStream drops at the end of this arm -> + // immediate close -> RST to the probing client. + } + }); + port + } + + /// Seed an isolated repos.json with `proj` registered, and isolate both + /// env vars the remove path reads (config path + serve port). The + /// returned guard restores the previous values when the test ends. + fn seed_repos_config( + tmp: &std::path::Path, + proj: &std::path::Path, + serve_port: u16, + ) -> EnvRestore { + let cfg_path = tmp.join("repos.json"); + // Env vars FIRST, then save: `ReposConfig::save()` resolves its + // destination the same way `load()` does (env override, else the + // global default). Saving before the override is set writes the + // seed into the developer's REAL ~/.codesearch/repos.json — which + // is exactly how this test once destroyed a registry during + // development. The canary below pins that invariant. + // + // SERVE_HOST_ENV is pinned too: `try_delegate_rm_to_serve` resolves + // the probe host via `resolve_serve_host()` — a machine with the var + // set would send the delegation somewhere other than the 127.0.0.1 + // listener these tests bind (same trap the serve e2e test documents). + let guard = EnvRestore::set(&[ + ( + crate::constants::REPOS_CONFIG_ENV, + &cfg_path.to_string_lossy(), + ), + ( + crate::constants::SERVE_PORT_ENV, + serve_port.to_string().as_str(), + ), + (crate::constants::SERVE_HOST_ENV, "127.0.0.1"), + ]); + let mut cfg = ReposConfig::default(); + cfg.register(proj.to_path_buf()); + cfg.save().expect("seed repos.json save must succeed"); + // Paranoia guard: the seed MUST have landed in the temp dir. + assert!( + cfg_path.exists(), + "seed repos.json must be written to the temp path, not the global default" + ); + guard + } + + /// Snapshot of the developer's REAL global repos.json (if present). + /// Tests assert it is byte-identical when they finish — a mis-ordered + /// seed or a stray save must fail the test, not silently destroy the + /// registry. + fn global_config_canary() -> Option { + let home = std::env::var("USERPROFILE") + .or_else(|_| std::env::var("HOME")) + .unwrap_or_default(); + if home.is_empty() { + return None; + } + let p = std::path::Path::new(&home) + .join(".codesearch") + .join("repos.json"); + std::fs::read_to_string(p).ok() + } + + fn assert_global_config_unchanged(before: Option) { + let now = global_config_canary(); + assert_eq!( + before, now, + "the developer's global repos.json changed during the test — \ + a save bypassed the CODESEARCH_REPOS_CONFIG override" + ); + } + + fn registered_paths() -> Vec { + ReposConfig::load() + .expect("repos.json load must succeed") + .repos + .values() + .map(|p| p.to_string_lossy().to_string()) + .collect() + } + + /// The actual defect: a delete that FAILS must leave repos.json + /// untouched. `.codesearch.db` as a FILE (not a directory) makes + /// `fs::remove_dir_all` fail deterministically on every platform — + /// standing in for a locked LMDB dir. + #[tokio::test] + #[serial] + async fn failed_delete_leaves_repos_json_untouched() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); + std::fs::write(proj.join(".codesearch.db"), b"not a directory").unwrap(); + let _env = seed_repos_config(tmp.path(), &proj, spawn_reset_server().await); + let _canary = global_config_canary(); + + let result = remove_from_index(Some(proj.clone()), false).await; + assert!( + result.is_err(), + "remove must fail when the db dir cannot be deleted" + ); + + // The entry must STILL be registered — the whole point of the fix. + let paths = registered_paths(); + assert!( + paths.iter().any(|p| Path::new(p) == proj), + "repos.json must be untouched after a failed delete, got: {:?}", + paths + ); + // And the db file itself is still there (nothing half-deleted). + assert!( + proj.join(".codesearch.db").exists(), + "the db path must still exist" + ); + assert_global_config_unchanged(_canary); + } + + /// Happy path: files removed AND entry unregistered, in that order. + #[tokio::test] + #[serial] + async fn successful_delete_unregisters_from_repos_json() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); + let db = proj.join(".codesearch.db"); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write(db.join("data.mdb"), b"fake").unwrap(); + let _env = seed_repos_config(tmp.path(), &proj, spawn_reset_server().await); + let _canary = global_config_canary(); + + remove_from_index(Some(proj.clone()), false) + .await + .expect("remove must succeed when nothing locks the db"); + + assert!(!db.exists(), "db dir must be gone"); + let paths = registered_paths(); + assert!( + !paths.iter().any(|p| Path::new(p) == proj), + "entry must be unregistered after successful delete, got: {:?}", + paths + ); + assert_global_config_unchanged(_canary); + } + + /// `--keep-config`: files removed, entry preserved. + #[tokio::test] + #[serial] + async fn keep_config_removes_files_but_preserves_entry() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); + let db = proj.join(".codesearch.db"); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write(db.join("data.mdb"), b"fake").unwrap(); + let _env = seed_repos_config(tmp.path(), &proj, spawn_reset_server().await); + let _canary = global_config_canary(); + + remove_from_index(Some(proj.clone()), true) + .await + .expect("keep-config remove must succeed"); + + assert!(!db.exists(), "db dir must be gone"); + let paths = registered_paths(); + assert!( + paths.iter().any(|p| Path::new(p) == proj), + "entry must be PRESERVED with keep_config, got: {:?}", + paths + ); + assert_global_config_unchanged(_canary); + } + + /// Global-only WITH `--keep-config`: the entry must survive. Before the + /// todo #48 rework this quadrant ran `unregister_repository(..)?` in an + /// `else if has_global` arm that never consulted `keep_config`, so the + /// flag was silently ignored and the entry was removed anyway. + #[tokio::test] + #[serial] + async fn keep_config_global_only_preserves_entry() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); // no .codesearch.db at all + let _env = seed_repos_config(tmp.path(), &proj, spawn_reset_server().await); + let _canary = global_config_canary(); + + remove_from_index(Some(proj.clone()), true) + .await + .expect("keep-config global-only remove must succeed"); + + let paths = registered_paths(); + assert!( + paths.iter().any(|p| Path::new(p) == proj), + "entry must be PRESERVED with keep_config in the global-only case, got: {:?}", + paths + ); + assert_global_config_unchanged(_canary); + } + + /// Global-only (no local db, only a repos.json entry): unregister works, + /// previously via a redundant second unregister call. + #[tokio::test] + #[serial] + async fn global_only_unregisters_entry() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); // no .codesearch.db at all + let _env = seed_repos_config(tmp.path(), &proj, spawn_reset_server().await); + let _canary = global_config_canary(); + + remove_from_index(Some(proj.clone()), false) + .await + .expect("global-only remove must succeed"); + + let paths = registered_paths(); + assert!( + !paths.iter().any(|p| Path::new(p) == proj), + "entry must be unregistered in global-only case, got: {:?}", + paths + ); + assert_global_config_unchanged(_canary); + } + + /// Serve answered 200 but honestly reported `removed_db_locked` — the + /// delegated removal must CARRY that outcome (`db_deleted == false` plus + /// the reason), never flatten it to plain success. The old delegation + /// checked only `status().is_success()`, so the CLI printed "DB deleted." + /// for files that were still on disk (BUG2 class, at the IPC boundary). + #[tokio::test] + #[serial] + async fn delegated_removal_carries_locked_db_outcome() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "proj"); + let db = proj.join(".codesearch.db"); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write(db.join("data.mdb"), b"fake").unwrap(); + + // A stand-in serve answering what `remove_repo_handler` emits when + // the lock-class retry budget is exhausted: 200 + honest payload. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = axum::Router::new() + .route( + "/health", + axum::routing::get(|| async { + axum::Json(serde_json::json!({ "codesearch_server": true })) + }), + ) + .route( + "/repos/{alias}", + axum::routing::delete( + |axum::extract::Path(alias): axum::extract::Path| async move { + axum::Json(serde_json::json!({ + "status": "removed_db_locked", + "alias": alias, + "db_deleted": false, + "db_delete_error": "mock: dir still held by a transient store holder", + })) + }, + ), + ); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + // Bounded readiness wait (same pattern as the serve e2e test — take + // `.port()`, never the SocketAddr, or URLs silently degrade). + for _ in 0..200 { + if tokio::net::TcpStream::connect(format!("127.0.0.1:{port}")) + .await + .is_ok() + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + + let _env = seed_repos_config(tmp.path(), &proj, port); + let _canary = global_config_canary(); + + let removed = super::try_delegate_rm_to_serve(&Some(proj.clone())) + .await + .expect("delegation must succeed against a 200-answering serve"); + + assert_eq!(removed.alias, "proj"); + assert_eq!(removed.project_path, proj); + assert!( + !removed.db_deleted, + "the locked-DB outcome must be carried, not flattened to success" + ); + assert_eq!( + removed.db_delete_error.as_deref(), + Some("mock: dir still held by a transient store holder"), + "serve's delete-error reason must survive the IPC boundary" + ); + // The stand-in left the files alone, exactly like a locked serve. + assert!(db.exists(), "db dir must still be on disk in this scenario"); + assert_global_config_unchanged(_canary); + } + + /// The DELETE request must wait out serve's legitimate slow removal + /// instead of firing the CLI's own 3 s health-probe timeout + /// mid-removal (todo #48 L2: "file-delete succeeds without + /// serve-stop"). A stand-in serve whose DELETE handler sleeps 4 s — + /// past the old client timeout, far under the new budget-derived one — + /// then answers the honest payload: delegation must succeed and carry + /// the outcome. Under the old shared 3 s client this test fails with + /// "delete failed: operation timed out" (mutation-verified), which in + /// production dropped the CLI onto the local path whose delete then + /// failed on the files serve was still tearing down. + #[tokio::test] + #[serial] + async fn delegated_rm_delete_outlives_slow_serve_removal() { + let tmp = tempfile::tempdir().unwrap(); + let proj = make_proj(tmp.path(), "slowproj"); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + // 4 s: comfortably past the old 3 s client timeout, comfortably + // under the new one (60 + 2*5 + 10 = 80 s) so the test stays fast. + let app = axum::Router::new() + .route( + "/health", + axum::routing::get(|| async { + axum::Json(serde_json::json!({ "codesearch_server": true })) + }), + ) + .route( + "/repos/{alias}", + axum::routing::delete( + |axum::extract::Path(alias): axum::extract::Path| async move { + tokio::time::sleep(std::time::Duration::from_secs(4)).await; + axum::Json(serde_json::json!({ + "status": "removed", + "alias": alias, + "db_deleted": true, + "db_delete_error": serde_json::Value::Null, + })) + }, + ), + ); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + // Bounded readiness wait (same pattern as the serve e2e test — take + // `.port()`, never the SocketAddr, or URLs silently degrade). + for _ in 0..200 { + if tokio::net::TcpStream::connect(format!("127.0.0.1:{port}")) + .await + .is_ok() + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + + let _env = seed_repos_config(tmp.path(), &proj, port); + let _canary = global_config_canary(); + + let removed = super::try_delegate_rm_to_serve(&Some(proj.clone())) + .await + .expect("the DELETE must outlive a slow serve removal, not time out at 3 s"); + + assert_eq!(removed.alias, "slowproj"); + assert!( + removed.db_deleted, + "the slow-but-successful outcome must be carried" + ); + assert_global_config_unchanged(_canary); + } +} + +/// Model resolution for the CLI `index` path: the model recorded in the index's +/// `metadata.json` must win, so re-indexing a non-default index (e.g. rebuilt +/// with EmbeddingGemma) does not silently downgrade it to 384-dim MiniLM. +/// +/// Regression guard for `index_with_options` using `model.unwrap_or_default()`: +/// with the defect reintroduced, `resolve_index_model` returns the default for +/// the gemma case below and the test fails. +#[cfg(test)] +mod index_model_resolution_tests { + use super::*; + + /// Write an index metadata.json recording `model` (as the indexer does). + fn write_metadata(db_path: &Path, model: ModelType) { + std::fs::create_dir_all(db_path).unwrap(); + let mut obj = serde_json::Map::new(); + model.write_metadata_fields(&mut obj); + std::fs::write( + db_path.join("metadata.json"), + serde_json::to_string(&obj).unwrap(), + ) + .unwrap(); + } + + #[test] + fn recorded_dimensions_reads_metadata_dimensions() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join(".codesearch.db"); + write_metadata(&db, ModelType::EmbeddingGemma300MQ4); + assert_eq!(recorded_dimensions(&db), 768); + } + + #[test] + fn recorded_dimensions_falls_back_to_model_then_default() { + // Dimensions key absent, model known -> model's dimensions. + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join(".codesearch.db"); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write( + db.join("metadata.json"), + r#"{"model_short_name":"embeddinggemma-q4"}"#, + ) + .unwrap(); + assert_eq!(recorded_dimensions(&db), 768); + + // No metadata at all -> default. + let dir = tempfile::tempdir().unwrap(); + assert_eq!( + recorded_dimensions(&dir.path().join(".codesearch.db")), + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS + ); + } + + #[test] + fn resolve_index_model_prefers_recorded_model_on_existing_index() { + // The defect: a bare `codesearch index` on a gemma index picked the + // default (384-dim MiniLM). The recorded model must win. + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join(".codesearch.db"); + write_metadata(&db, ModelType::EmbeddingGemma300MQ4); + + assert_eq!( + resolve_index_model(&db, false, None).unwrap(), + ModelType::EmbeddingGemma300MQ4, + "an existing index's recorded model must be used, not the default" + ); + // An explicit --model that agrees is accepted. + assert_eq!( + resolve_index_model(&db, false, Some(ModelType::EmbeddingGemma300MQ4)).unwrap(), + ModelType::EmbeddingGemma300MQ4 + ); + } + + #[test] + fn resolve_index_model_rejects_disagreeing_override_without_force() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join(".codesearch.db"); + write_metadata(&db, ModelType::EmbeddingGemma300MQ4); + + let err = resolve_index_model(&db, false, Some(ModelType::AllMiniLML6V2Q)) + .expect_err("a --model that disagrees with the index must be rejected"); + let msg = format!("{err:#}"); + assert!( + msg.contains("Model mismatch") && msg.contains("embeddinggemma-q4"), + "error must name the recorded model and the mismatch, got: {msg}" + ); + + // --force deletes the DB, so the requested model is honoured. + assert_eq!( + resolve_index_model(&db, true, Some(ModelType::AllMiniLML6V2Q)).unwrap(), + ModelType::AllMiniLML6V2Q + ); + } + + #[test] + fn resolve_index_model_uses_requested_model_for_a_fresh_index() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join(".codesearch.db"); // does not exist + + assert_eq!( + resolve_index_model(&db, false, Some(ModelType::EmbeddingGemma300MQ4)).unwrap(), + ModelType::EmbeddingGemma300MQ4 + ); + assert_eq!( + resolve_index_model(&db, false, None).unwrap(), + ModelType::default() + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 1a0be48d..f50f2ae3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,8 @@ pub mod rerank; pub mod search; pub mod serve; pub mod symbols; +#[cfg(test)] +pub mod testing; pub mod utils; pub mod vectordb; pub mod watch; diff --git a/src/lmdb_registry.rs b/src/lmdb_registry.rs index 81d4092d..1f4cd65a 100644 --- a/src/lmdb_registry.rs +++ b/src/lmdb_registry.rs @@ -14,7 +14,7 @@ use dashmap::DashMap; use std::mem::ManuallyDrop; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock, Weak}; use std::time::Instant; use crate::cache::safe_canonicalize; @@ -42,6 +42,11 @@ use crate::cache::safe_canonicalize; /// Because heed refuses to reopen the same path with different options, this /// must be applied at EVERY env-open site, not only the read-only one — a /// partial rollout would turn a working reopen into an intermittent failure. +// heed 0.22 deprecates the NO_TLS flag in favour of the type-state +// `EnvOpenOptions::read_txn_without_tls()`, but that changes `Env`'s generic +// parameter across the whole registry and the txn Send semantics with it. +// MDB_NOTLS itself is unchanged in LMDB — same flag, same behavior. +#[allow(deprecated)] pub const BASE_ENV_FLAGS: heed::EnvFlags = heed::EnvFlags::NO_TLS; // ── Global registry ───────────────────────────────────────────── @@ -110,6 +115,107 @@ pub fn is_open(path: &Path) -> bool { } } +/// Descriptions of every live [`TrackedEnv`] whose canonical path is `path` +/// itself or lives UNDER it (component-wise prefix, so `base/foo` does not +/// match `base/foobar`). An empty `Vec` means no in-process holder keeps any +/// LMDB env open anywhere in that subtree. +/// +/// This is the single source of truth for "can this DB directory be deleted +/// by this process right now": it sees through every holder shape — an outer +/// `Arc` clone held by an in-flight search, an inner +/// `Arc>` captured by a `spawn_blocking` embed pass, a +/// `SCIP(...)` env in a `scip/` subdirectory — because all of them keep their +/// `TrackedEnv` (and therefore its registry slot) alive. `remove_repo` waits +/// on this before retrying a locked delete. +/// +/// A path that cannot be canonicalized (e.g. already deleted) reports no +/// holders — the safe "nothing left to wait for" answer. +pub fn open_holders_under(path: &Path) -> Vec { + let canonical = match safe_canonicalize(path) { + Ok(c) => c, + Err(_) => return Vec::new(), + }; + match LMDB_REGISTRY.get() { + Some(registry) => registry + .iter() + .filter(|entry| entry.key().starts_with(&canonical)) + .map(|entry| entry.value().description.clone()) + .collect(), + None => Vec::new(), + } +} + +// ── Shared-env cache ──────────────────────────────────────────── + +/// Process-wide cache of LMDB environments that multiple components must use +/// CONCURRENTLY (queries, rebuilds, per-language adapters on the same +/// directory). Holds only [`Weak`] references: the cache never keeps an +/// environment alive, it just hands out the live one when it exists. When the +/// last user drops their `Arc`, the [`TrackedEnv`] drops, its registry slot +/// frees, and the stale cache entry is reaped on the next lookup. +static SHARED_ENVS: OnceLock>> = OnceLock::new(); + +/// Open the environment at `path`, or return the already-open shared instance. +/// +/// LMDB permits exactly one open environment per directory per process, so +/// callers that may overlap in time (a rebuild vs. an in-flight query, the C# +/// vs. TypeScript adapters on the same `db_path/scip`) must not each open +/// their own — the second [`TrackedEnv::open`] trips the double-open guard and +/// one side fails outright. This getter makes the collision impossible: the +/// first caller opens (running `init` once to create the named databases) and +/// everyone else receives a clone of the same `Arc`. +/// +/// `build_opts` configures the [`heed::EnvOpenOptions`] and `init` runs once +/// per environment lifetime, right after the open, before the handle is +/// published to other threads. Writers then serialise on LMDB's own +/// single-writer mutex and readers never block. +/// +/// Both closures run while the cache's shard lock is held: they must not +/// re-enter `get_or_open_shared_env` (a path hashing to the same shard +/// self-deadlocks) and should stay cheap. +/// +/// The env-var lookups inside `build_opts`/`init` run only when the directory +/// is opened for the first time in this process — later override changes do +/// not affect an already-shared environment. +pub fn get_or_open_shared_env( + path: &Path, + description: &str, + build_opts: impl FnOnce(&mut heed::EnvOpenOptions), + init: impl FnOnce(&TrackedEnv) -> Result<()>, +) -> Result> { + let canonical = safe_canonicalize(path) + .with_context(|| format!("Cannot canonicalize LMDB path: {}", path.display()))?; + let cache = SHARED_ENVS.get_or_init(DashMap::new); + + loop { + use dashmap::mapref::entry::Entry; + match cache.entry(canonical.clone()) { + Entry::Occupied(occupied) => { + if let Some(env) = occupied.get().upgrade() { + return Ok(env); + } + // Last Arc dropped but the entry survived — reap and retry so + // the Vacant arm below performs a fresh open. + occupied.remove(); + } + Entry::Vacant(vacant) => { + let mut opts = heed::EnvOpenOptions::new(); + build_opts(&mut opts); + // SAFETY: caller contract — same as `TrackedEnv::open`. The + // registry slot guards against any concurrent direct open. + let tracked = unsafe { TrackedEnv::open(&opts, path, description)? }; + // Run init before publishing: the `?` early-return drops + // `tracked`, freeing the registry slot, so a failed init + // leaves the path openable. + init(&tracked)?; + let env = Arc::new(tracked); + vacant.insert(Arc::downgrade(&env)); + return Ok(env); + } + } + } +} + // ── TrackedEnv wrapper ────────────────────────────────────────── /// Wrapper around [`heed::Env`] that prevents double-open panics. @@ -156,10 +262,12 @@ impl TrackedEnv { impl Drop for TrackedEnv { fn drop(&mut self) { - // Ordering here is load-bearing. heed maintains its OWN process-global - // registry of opened environments (`OPENED_ENV`), keyed by canonical - // path, that outlives a `heed::Env` until its last strong ref drops. - // If we `unregister()` from our registry FIRST and let the field drop + // Ordering here is load-bearing, twice over. + // + // (1) The env must be closed BEFORE we free our own registry slot. + // heed maintains its OWN process-global registry of opened + // environments (`OPENED_ENV`), keyed by canonical path. If we + // `unregister()` from our registry FIRST and let the field drop // afterwards (the default Rust drop order: body, then fields), there is // a window where our slot is free but heed's env is still alive. A // concurrent `TrackedEnv::open` on the same path — e.g. the idle reaper @@ -168,18 +276,37 @@ impl Drop for TrackedEnv { // rejects with the cryptic "an environment is already opened with // different options" (once a prior MDB_MAP_FULL resize left the live // env's recorded map_size differing from the reopen's resolved size). + // Closing the env before `unregister()` enforces the invariant + // "our slot free ⟹ heed's slot free": a concurrent open either sees + // our slot still occupied (clear "double-open prevented" + retry) or + // sees both free (clean reopen). It can never observe the inconsistent + // state that produces heed's raw error. // - // Dropping the `heed::Env` BEFORE `unregister()` enforces the invariant - // "our slot free ⟹ heed's slot free": a concurrent open either sees our - // slot still occupied (clear "double-open prevented" + retry) or sees - // both free (clean reopen). It can never observe the inconsistent state - // that produces heed's raw error. + // (2) A plain drop of the `heed::Env` does NOT close the environment. + // heed 0.20's `OPENED_ENV` entry itself holds a strong `Env` clone + // (`EnvEntry { env: Some(env.clone()), .. }` — inserted at open, used + // to hand out further clones on re-open). With our wrapper as the only + // user-side reference, dropping it leaves the Arc count at exactly 1: + // the entry's own clone. `EnvInner::drop` — and with it + // `mdb_env_close` — therefore NEVER runs. On POSIX that leaks an fd + // and an mmap silently; on Windows it locks `data.mdb`/`lock.mdb` + // against deletion for the lifetime of the process, which is why + // `index rm` against a running serve could not delete the DB dir + // (deterministic os error 32 after the whole retry budget, LMDB + // registry long empty — the holder is invisible to it). + // `prepare_for_closing()` is heed's one real close path: it takes the + // entry's reference out and drops the last one synchronously + // (entry removed, `mdb_env_close` called, waiters signalled) before + // returning. It is correct here because nothing in this codebase + // clones the `heed::Env` out of a `TrackedEnv` (the deref only lends + // `&Env`; `TrackedEnv` itself is not `Clone`), so this wrapper holds + // the last user-side reference. // - // SAFETY: `inner` is dropped exactly once, here, and never accessed - // again (the surrounding `TrackedEnv` is being destroyed). - unsafe { - ManuallyDrop::drop(&mut self.inner); - } + // SAFETY: `inner` is taken exactly once, here, and the `ManuallyDrop` + // slot is never touched again afterwards (the surrounding + // `TrackedEnv` is being destroyed). + let env = unsafe { ManuallyDrop::take(&mut self.inner) }; + env.prepare_for_closing(); unregister(&self.canonical); } } @@ -213,6 +340,10 @@ mod tests { fn make_opts_sized(map_size: usize) -> heed::EnvOpenOptions { let mut opts = heed::EnvOpenOptions::new(); opts.map_size(map_size).max_dbs(1); + // Every test open carries the baseline flags per the AGENTS.md rule + // ("open every env with BASE_ENV_FLAGS") so new tests cannot drift + // from production open options. + unsafe { opts.flags(BASE_ENV_FLAGS) }; opts } @@ -254,7 +385,6 @@ mod tests { assert!(err.contains("double-open prevented")); assert!(err.contains("test-1")); } - #[test] fn test_registry_allows_reopen_after_drop() { let dir = TempDir::new().unwrap(); @@ -270,6 +400,60 @@ mod tests { let _env2 = unsafe { TrackedEnv::open(&opts, path, "test-2").unwrap() }; } + /// Dropping the last `TrackedEnv` must REALLY close the heed environment. + /// + /// heed 0.20's `OPENED_ENV` entry holds a strong `Env` clone of its own, + /// so a plain drop of the user-side `Env` leaves the Arc count at 1 (the + /// entry's) and `mdb_env_close` never runs — the env stays open invisibly: + /// `env_closing_event` keeps answering `Some`, and on Windows `data.mdb`/ + /// `lock.mdb` stay locked against deletion for the life of the process + /// (the deterministic `index rm` os-error-32 failure this test pins). + /// `TrackedEnv::drop` therefore closes via `prepare_for_closing()`. + /// + /// Cross-platform: the `env_closing_event` assert fails everywhere when + /// the close path regresses; the directory-delete assert is the + /// Windows-visible consequence of the same leak and guards it directly. + #[test] + fn drop_really_closes_heed_env_and_releases_the_files() { + let dir = TempDir::new().unwrap(); + let db_path = dir.path().join("db"); + std::fs::create_dir(&db_path).unwrap(); + // heed's OPENED_ENV is keyed by its own `canonicalize_path`, and the + // query in `env_closing_event` is an EXACT un-normalized lookup. On + // Windows heed canonicalizes and then round-trips through a file URL, + // which strips the `\\?\` UNC prefix — the key is the plain long + // path. A raw TempDir path here can be an 8.3 short name + // (`DEVELT~1`), which normalizes to a different string than the + // query, so the lookup misses and the test fails on Windows while + // passing on Linux (where both forms are identical). `safe_canonicalize` + // (canonicalize + UNC-strip) is exactly heed's Windows normalization, + // so the query matches the registry key on every OS. + let heed_key = safe_canonicalize(&db_path).unwrap(); + let opts = make_opts(); + + { + let _env = unsafe { TrackedEnv::open(&opts, &db_path, "close-on-drop-test").unwrap() }; + assert!( + heed::env_closing_event(&heed_key).is_some(), + "while the TrackedEnv lives, heed must report the env open" + ); + } + + // The env must now be gone from heed's own registry too — not just + // from ours (`open_holders_under` is vacuous here, it tracks + // TrackedEnvs only). + assert!( + heed::env_closing_event(&heed_key).is_none(), + "heed's OPENED_ENV entry must be removed on TrackedEnv drop; \ + a surviving entry means mdb_env_close never ran" + ); + + // ...which is what makes the DB directory deletable on Windows. + std::fs::remove_dir_all(&db_path) + .expect("db dir must be deletable after the last TrackedEnv drops"); + assert!(!db_path.exists()); + } + #[test] fn test_different_paths_both_allowed() { let dir1 = TempDir::new().unwrap(); @@ -280,6 +464,182 @@ mod tests { let _env2 = unsafe { TrackedEnv::open(&opts, dir2.path(), "test-2").unwrap() }; } + fn shared_opts(opts: &mut heed::EnvOpenOptions) { + opts.map_size(1024 * 1024).max_dbs(4); + // Every test open carries the baseline flags per the AGENTS.md rule. + unsafe { opts.flags(BASE_ENV_FLAGS) }; + } + + /// Direct-open options IDENTICAL to `shared_opts`. heed refuses to reopen + /// a path with different options (max_dbs included) even after the prior + /// env dropped — the AGENTS.md "same options on every open" rule — so the + /// direct opens in the shared-env tests must not reuse `make_opts` + /// (max_dbs 1). + fn shared_compatible_opts() -> heed::EnvOpenOptions { + let mut opts = heed::EnvOpenOptions::new(); + shared_opts(&mut opts); + opts + } + + /// Two callers of the shared getter receive the SAME environment — this is + /// the property whose absence made a watcher rebuild fail with + /// `LMDB double-open prevented` while a lazy find-refs held its own env. + #[test] + fn shared_env_returns_live_instance_to_concurrent_callers() { + let dir = TempDir::new().unwrap(); + let path = dir.path().to_path_buf(); + + let env1 = get_or_open_shared_env(&path, "shared-1", shared_opts, |_| Ok(())).unwrap(); + let env2 = get_or_open_shared_env(&path, "shared-2", shared_opts, |_| Ok(())).unwrap(); + assert!(Arc::ptr_eq(&env1, &env2)); + + // While the shared env is alive, a DIRECT open on the same path must + // still trip the guard — the shared env genuinely occupies the slot. + let direct = unsafe { TrackedEnv::open(&shared_compatible_opts(), &path, "direct") }; + let err = direct.unwrap_err().to_string(); + assert!(err.contains("double-open prevented")); + assert!(err.contains("shared-1")); + } + + /// The cache holds only weak refs: once the last user drops their Arc the + /// registry slot frees (a direct open succeeds) and the next shared caller + /// transparently reopens. + #[test] + fn shared_env_reopens_after_all_arcs_drop() { + let dir = TempDir::new().unwrap(); + let path = dir.path().to_path_buf(); + + { + let _env = get_or_open_shared_env(&path, "shared-1", shared_opts, |_| Ok(())).unwrap(); + } + + // Slot freed after the last Arc dropped. + { + let _direct = + unsafe { TrackedEnv::open(&shared_compatible_opts(), &path, "direct").unwrap() }; + } + + // And the shared getter opens fresh again. + let _again = get_or_open_shared_env(&path, "shared-2", shared_opts, |_| Ok(())).unwrap(); + } + + /// An `init` failure must not leak the registry slot: the error propagates + /// and the env is dropped, leaving the path openable. + #[test] + fn shared_env_init_failure_frees_slot() { + let dir = TempDir::new().unwrap(); + let path = dir.path().to_path_buf(); + + let result = get_or_open_shared_env(&path, "shared-1", shared_opts, |_| { + Err(anyhow::anyhow!("boom")) + }); + assert!(result.is_err()); + + let _direct = + unsafe { TrackedEnv::open(&shared_compatible_opts(), &path, "direct").unwrap() }; + } + + /// N threads racing the FIRST open all succeed and all hold the same env — + /// no thread sees the double-open error the per-caller open produced. + #[test] + fn shared_env_concurrent_first_open_is_safe() { + let dir = TempDir::new().unwrap(); + let path = Arc::new(dir.path().to_path_buf()); + + let handles: Vec<_> = (0..8) + .map(|_| { + let p = Arc::clone(&path); + std::thread::spawn(move || { + get_or_open_shared_env(&p, "shared-race", shared_opts, |_| Ok(())) + }) + }) + .collect(); + + let envs: Vec<_> = handles + .into_iter() + .map(|h| h.join().expect("thread panicked").expect("open failed")) + .collect(); + assert!(envs.windows(2).all(|w| Arc::ptr_eq(&w[0], &w[1]))); + } + + /// `open_holders_under` reports every live env at or under the queried + /// path — the precondition check `remove_repo`'s lock-class retry waits + /// on. Must see (a) an env whose path IS the queried path and (b) an env + /// in a SUBDIRECTORY (the `scip/` case), with each holder's description + /// carried for triage. + #[test] + fn open_holders_under_reports_envs_at_and_below_path() { + let tmp = TempDir::new().unwrap(); + let db_dir = tmp.path().join("proj").join(".codesearch.db"); + std::fs::create_dir_all(db_dir.join("scip")).unwrap(); + let opts = make_opts(); + + let _env_db = unsafe { TrackedEnv::open(&opts, &db_dir, "SharedStores(proj)").unwrap() }; + let _env_scip = + unsafe { TrackedEnv::open(&opts, &db_dir.join("scip"), "SCIP(proj)").unwrap() }; + + let holders = open_holders_under(&tmp.path().join("proj").join(".codesearch.db")); + assert_eq!( + holders.len(), + 2, + "both the db-dir env and the scip-subdir env must be reported, got: {holders:?}" + ); + assert!(holders.contains(&"SharedStores(proj)".to_string())); + assert!(holders.contains(&"SCIP(proj)".to_string())); + } + + /// Component-boundary safety: `base/foobar` must NOT count as a holder + /// under `base/foo` — `Path::starts_with` is component-wise, and a string + /// prefix match here would make `remove_repo` wait on (and warn about) + /// holders of an entirely different repo's database. + #[test] + fn open_holders_under_does_not_match_partial_component() { + let tmp = TempDir::new().unwrap(); + let foo = tmp.path().join("foo"); + let foobar = tmp.path().join("foobar"); + std::fs::create_dir_all(&foo).unwrap(); + std::fs::create_dir_all(&foobar).unwrap(); + let opts = make_opts(); + + let _env = unsafe { TrackedEnv::open(&opts, &foobar, "other-repo").unwrap() }; + + assert!( + open_holders_under(&foo).is_empty(), + "env at a sibling path sharing a string prefix must not be reported" + ); + } + + /// The drain contract: once the last `TrackedEnv` drops, its registry slot + /// goes with it — an empty `Vec` is `remove_repo`'s "deletable now" signal + /// and must not lag behind the actual drop. + #[test] + fn open_holders_under_is_empty_after_env_drops() { + let dir = TempDir::new().unwrap(); + let opts = make_opts(); + + { + let _env = unsafe { TrackedEnv::open(&opts, dir.path(), "transient-search").unwrap() }; + assert!( + !open_holders_under(dir.path()).is_empty(), + "holder must be visible while the env is live" + ); + } + + assert!( + open_holders_under(dir.path()).is_empty(), + "holder must be gone after the env drops" + ); + } + + /// A path that cannot be canonicalized (e.g. already deleted by a + /// concurrent cleaner) reports no holders — the safe "nothing left to + /// wait for" answer, never a spurious error. + #[test] + fn open_holders_under_reports_none_for_missing_path() { + let missing = std::env::temp_dir().join("codesearch-never-exists-here"); + assert!(open_holders_under(&missing).is_empty()); + } + /// Regression guard for the concurrent open→drop→reopen path that produced /// the production 500 ("an environment is already opened with different /// options"). diff --git a/src/logger/mod.rs b/src/logger/mod.rs index f5cb094a..2281437b 100644 --- a/src/logger/mod.rs +++ b/src/logger/mod.rs @@ -108,11 +108,21 @@ pub fn ensure_log_dir(log_dir: &Path) -> Result<()> { /// Try to extract a date from a daily-rotated log filename. /// /// tracing-appender DAILY rotation produces files named `.YYYY-MM-DD`. -/// Returns `None` if the filename doesn't match the expected pattern. +/// Accepts both the CLI prefix (`codesearch.log.`) and the serve prefix +/// (`serve.log.`) — a cleanup pass over the shared global log directory must +/// retire both, which is exactly the gap that let 4 months of serve logs +/// pile up (183 files / 299 MB) while the code defaulted to 5 files / +/// 5 days. Returns `None` if the filename doesn't match either pattern. fn parse_log_date(file_name: &str) -> Option { - // Pattern: "codesearch.log.YYYY-MM-DD" - let suffix = file_name.strip_prefix(&format!("{}.", LOG_FILE_NAME))?; - NaiveDate::parse_from_str(suffix, "%Y-%m-%d").ok() + // Patterns: "codesearch.log.YYYY-MM-DD" / "serve.log.YYYY-MM-DD" + for prefix in [LOG_FILE_NAME, SERVE_LOG_FILE_NAME] { + if let Some(suffix) = file_name.strip_prefix(&format!("{}.", prefix)) { + if let Ok(date) = NaiveDate::parse_from_str(suffix, "%Y-%m-%d") { + return Some(date); + } + } + } + None } /// Remove old log files based on retention period and max file count. @@ -211,6 +221,13 @@ pub fn init_logger(db_path: &Path, log_level: LogLevel, quiet: bool) -> Result Result, + ) -> Result { + let kind = request.kind.as_deref().unwrap_or("outline").to_lowercase(); + tracing::info!( + "📥 explore(target={:?}, kind={}, project={:?})", + request.target, + kind, + request.project, + ); + match kind.as_str() { + "outline" => { + let outline_req = FileOutlineRequest { + path: request.target, + project: request.project, + group: request.group, + }; + self.file_outline(Parameters(outline_req)).await + } + "similar" => { + let chunk_id = match request.target.parse::() { + Ok(id) => id, + Err(_) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "For similar mode, `target` must be a numeric chunk_id, got: '{}'", + request.target + ))])); + } + }; + let similar_req = SimilarChunksRequest { + chunk_id, + limit: request.limit, + project: request.project, + group: request.group, + }; + self.similar_chunks(Parameters(similar_req)).await + } + _ => Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Unknown explore kind '{}'. Use `outline` or `similar`.", + kind + ))])), + } + } + + /// Fetch outline items for an already-normalised absolute path. + /// + /// Returns `Ok(vec![])` when no chunks match. + /// In multi-store mode, per-store I/O failures are recorded in `warnings` and + /// skipped (never `Err`) so one broken repo cannot blank the whole outline. + /// In single-store mode, I/O failures are returned as `Err`. + /// + /// `warnings` is not optional: without it a failed store is indistinguishable + /// from a file with no indexed chunks, and the caller is told the file is not + /// indexed — a diagnosis, and a wrong one. + async fn outline_items_for_normalized( + &self, + normalized: &str, + ctx: &MultiStoreContext, + warnings: &mut Vec, + ) -> anyhow::Result> { + if let Some(ref sv) = ctx.stores_vec { + let aliases = ctx.aliases(); + let mut all_items: Vec = Vec::new(); + let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.chunks_for_file(normalized) { + Ok(metas) => { + for c in metas { + if seen_ids.insert(c.id) { + all_items.push(FileOutlineItem { + chunk_id: c.id, + kind: c.kind, + signature: c.signature, + start_line: c.start_line, + end_line: c.end_line, + }); + } + } + } + Err(ref e) => { + note_store_failure(warnings, aliases, store_idx, "outline scan", e); + } + } + } + all_items.sort_by_key(|i| i.start_line); + Ok(all_items) + } else { + let normalized_owned = normalized.to_string(); + self.with_vector_store_read_for( + move |store| { + let mut out: Vec = store + .chunks_for_file(&normalized_owned)? + .into_iter() + .map(|c| FileOutlineItem { + chunk_id: c.id, + kind: c.kind, + signature: c.signature, + start_line: c.start_line, + end_line: c.end_line, + }) + .collect(); + out.sort_by_key(|i| i.start_line); + Ok(out) + }, + ctx.stores.clone(), + ) + .await + } + } + + async fn file_outline( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "explore") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + // Outline operates on a single repo — reject group fan-out + if ctx.is_multi { + return Ok(CallToolResult::success(vec![ContentBlock::text( + "Tool 'explore' operates on a single repo. Use 'project' instead of 'group'." + .to_string(), + )])); + } + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // In serve mode, use the resolved project root from alias_roots; + // self.project_path is "serve://multi-repo" which doesn't resolve. + let project_root = if let Some(ref alias) = ctx.project_alias { + ctx.alias_roots + .get(alias) + .map(PathBuf::from) + .unwrap_or_else(|| self.project_path.clone()) + } else { + self.project_path.clone() + }; + // Strip project-alias prefix from target path if present. + // E.g. "ExampleRepo/src/foo.cs" with project="ExampleRepo" → "src/foo.cs" + let stripped_path = strip_alias_prefix(&request.path, ctx.project_alias.as_ref()); + let normalized = normalize_tool_path(&stripped_path, &project_root); + + let mut outline_warnings: Vec = Vec::new(); + let mut items = match self + .outline_items_for_normalized(&normalized, &ctx, &mut outline_warnings) + .await + { + Ok(v) => v, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error reading outline: {e:#}" + ))])); + } + }; + + // Two-pass fallback: if alias-stripping changed the path and yielded no results, + // try the original un-stripped path. Handles the case where the project alias + // matches a package subdirectory name (e.g. project "my_pkg" with target + // "my_pkg/config.py" → after strip becomes "config.py" which is wrong; + // the correct relative path is "my_pkg/config.py"). + if items.is_empty() && stripped_path != request.path { + let normalized_orig = normalize_tool_path(&request.path, &project_root); + if normalized_orig != normalized { + tracing::debug!( + "file_outline: primary '{}' empty, trying fallback '{}'", + normalized, + normalized_orig + ); + items = match self + .outline_items_for_normalized(&normalized_orig, &ctx, &mut outline_warnings) + .await + { + Ok(v) => v, + Err(e) => { + tracing::warn!( + "file_outline: fallback '{}' also failed: {:?}", + normalized_orig, + e + ); + push_store_warning( + &mut outline_warnings, + &store_warning( + ctx.project_alias.as_deref().unwrap_or("unknown"), + "outline scan", + &format!("{e:#}"), + ), + ); + Vec::new() + } + }; + } + } + + respond_with_items(&items, &outline_warnings, || { + "No indexed chunks found for path. Verify the file is within the \ + project root and the index is up to date." + .to_string() + }) + } +} diff --git a/src/mcp/federation_helpers.rs b/src/mcp/federation_helpers.rs new file mode 100644 index 00000000..6fa8dd61 --- /dev/null +++ b/src/mcp/federation_helpers.rs @@ -0,0 +1,218 @@ +use super::types::SearchResultItem; +use rmcp::model::CallToolResult; + +// ════════════════════════════════════════════════════════════════ +// Federation helpers (module-scope) — merge / parse / convert. +// ════════════════════════════════════════════════════════════════ + +/// RRF-interleave several disjoint ranked lists into one ranked list. +/// +/// Each list is assumed already ranked best-first and disjoint from the others +/// (local repos vs. distinct remote peers). An item's merged score is +/// `1/(k + rank_in_own_list + 1)` (classic Reciprocal Rank Fusion with a `+1` +/// so the top hit never exceeds `1/k`). The union is sorted by score desc with a +/// stable source-order tiebreak, then truncated to `limit`. +pub(crate) fn merge_ranked_lists( + lists: Vec>, + k: f32, + limit: usize, +) -> Vec { + let mut merged: Vec<(f32, usize, SearchResultItem)> = Vec::new(); + let mut order = 0usize; + for list in lists { + for (rank, item) in list.into_iter().enumerate() { + let score = 1.0 / (k + rank as f32 + 1.0); + merged.push((score, order, item)); + order += 1; + } + } + // Sort by score desc; tiebreak on insertion order for stable, predictable + // output (local list first, then remotes in config order). + merged.sort_by(|a, b| { + b.0.partial_cmp(&a.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.1.cmp(&b.1)) + }); + merged + .into_iter() + .take(limit) + .map(|(score, _, mut it)| { + it.score = score; + it + }) + .collect() +} + +/// Extract the rendered tool payload from a `CallToolResult` and re-parse it as +/// local `SearchResultItem`s. Works for both semantic and literal modes — the +/// rendered JSON always has a top-level `results` array. +pub(crate) fn parse_search_items_from_call_result( + result: &CallToolResult, + mode: &str, +) -> Vec { + let text = extract_call_tool_text(result); + let value: serde_json::Value = match serde_json::from_str(&text) { + Ok(v) => v, + Err(_) => return Vec::new(), + }; + let results = match value.get("results").and_then(|r| r.as_array()) { + Some(arr) => arr, + None => return Vec::new(), + }; + match mode { + "semantic" => results + .iter() + .filter_map(|v| serde_json::from_value::(v.clone()).ok()) + .collect(), + // Literal items lack `chunk_id`; map their `snippet` into `content` so + // the merged list renders uniformly. The absent id stays `None` — + // fabricating `0` here once let callers build a bogus + // `/:0` chunk_ref that `get_chunk` silently resolved to + // an unrelated chunk. + _ => results + .iter() + .map(|v| SearchResultItem { + chunk_id: v.get("chunk_id").and_then(|c| c.as_u64()).map(|c| c as u32), + path: v + .get("path") + .and_then(|p| p.as_str()) + .unwrap_or("") + .to_string(), + start_line: v.get("start_line").and_then(|n| n.as_u64()).unwrap_or(0) as usize, + end_line: v.get("end_line").and_then(|n| n.as_u64()).unwrap_or(0) as usize, + kind: v + .get("kind") + .and_then(|k| k.as_str()) + .unwrap_or("") + .to_string(), + score: v.get("score").and_then(|s| s.as_f64()).unwrap_or(0.0) as f32, + signature: v + .get("signature") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()), + content: v + .get("snippet") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()), + context_prev: None, + context_next: None, + source: None, + chunk_ref: None, + }) + .collect(), + } +} + +/// Convert a remote search hit into a local `SearchResultItem`, tagging it with +/// its origin (`source`) and a project-namespaced `chunk_ref` for later +/// retrieval. +/// +/// The `chunk_ref` is `"/:"`. The `remote_alias` +/// segment is essential: the peer is itself multi-repo and chunk_ids are only +/// unique *within* a single index, so `federated_get_chunk` must forward the +/// alias as a `project=` scope to disambiguate. Omitting it (the old +/// `":"` shape) made every remote `get_chunk` fail with +/// `ambiguous_chunk_id` whenever the peer hosted more than one project. +/// +/// Literal hits carry no chunk id: both `chunk_id` and `chunk_ref` stay +/// `None` (the fields are omitted from the rendered JSON). Never substitute +/// a default here — a fabricated `chunk_id: 0` invites callers to hand-build +/// a `"/:0"` ref that resolves to an unrelated chunk. +pub(crate) fn convert_remote_item( + peer_name: &str, + remote_alias: &str, + item: crate::federation::RemoteSearchItem, +) -> SearchResultItem { + let chunk_ref = item + .chunk_id + .map(|id| format!("{peer_name}/{remote_alias}:{id}")); + SearchResultItem { + chunk_id: item.chunk_id, + path: item.path, + start_line: item.start_line, + end_line: item.end_line, + kind: item.kind.unwrap_or_default(), + score: item.score, + signature: item.signature, + content: item.content.or(item.snippet), + context_prev: item.context_prev, + context_next: item.context_next, + source: Some(format!("{peer_name}/{remote_alias}")), + chunk_ref, + } +} + +/// Apply a `filter_path` prefix filter to federated results **client-side**, +/// on the namespaced paths the caller actually sees. +/// +/// Federated `filter_path` cannot be forwarded to the peer: the peer matches +/// against its own un-namespaced store paths (and, in serve mode, against the +/// wrong project root), so a server-side match returns nothing for any value. +/// Here we match against the `//…` path carried on each converted +/// item, with an empty project root (the namespaced path is already relative), +/// so the filter means exactly what the caller reads back in the results. +/// +/// A blank/whitespace filter is a no-op. Returns immediately when `filter_path` +/// is `None`, so the non-filtered fast path pays nothing. +pub(crate) fn retain_by_filter_path(items: &mut Vec, filter_path: Option<&str>) { + let Some(raw) = filter_path else { return }; + if raw.trim().is_empty() { + return; + } + let normalized = crate::cache::normalize_filter_path(raw); + if normalized.is_empty() { + return; + } + items.retain(|it| crate::cache::path_matches_filter(&it.path, &normalized, "")); +} + +/// True when `filter_path` carries a meaningful prefix (non-blank, non-empty +/// after normalization) — the single predicate the federated search paths use +/// to decide whether to over-fetch and post-filter. Mirrors the no-op guards in +/// [`retain_by_filter_path`] so `has_filter` and the retain stay in lockstep. +pub(crate) fn is_meaningful_filter(filter_path: Option<&str>) -> bool { + filter_path + .map(|f| !f.trim().is_empty() && !crate::cache::normalize_filter_path(f).is_empty()) + .unwrap_or(false) +} + +/// Parse a federated `chunk_ref` into its `(peer, remote_alias, chunk_id)` +/// parts. +/// +/// Accepts the current project-namespaced shape `"/:"` and, +/// for backward compatibility, the legacy `":"` shape (no alias → +/// `None`, which falls back to group-scoped lookup on the peer). +/// +/// The `chunk_id` is taken after the *last* `':'` so peer/alias segments that +/// themselves contain a colon are not misparsed; the peer/alias split is on the +/// *first* `'/'`. +pub(crate) fn parse_federated_chunk_ref(chunk_ref: &str) -> Option<(&str, Option<&str>, u32)> { + let (left, id_str) = chunk_ref.rsplit_once(':')?; + let chunk_id: u32 = id_str.parse().ok()?; + match left.split_once('/') { + Some((peer, alias)) if !peer.is_empty() && !alias.is_empty() => { + Some((peer, Some(alias), chunk_id)) + } + _ => Some((left, None, chunk_id)), + } +} + +/// Best-effort extraction of the concatenated text content of a +/// `CallToolResult`. Resilient to rmcp's internal content enum shape. +pub(crate) fn extract_call_tool_text(result: &CallToolResult) -> String { + serde_json::to_value(result) + .ok() + .and_then(|v| { + v.get("content").and_then(|c| c.as_array()).map(|arr| { + arr.iter() + .filter_map(|item| item.get("text").and_then(|t| t.as_str())) + .collect::>() + .join("\n") + }) + }) + .unwrap_or_default() +} + +#[cfg(test)] +#[path = "federation_helpers_tests.rs"] +mod federation_helpers_tests; diff --git a/src/mcp/federation_helpers_tests.rs b/src/mcp/federation_helpers_tests.rs index 7c84dc92..01491d7d 100644 --- a/src/mcp/federation_helpers_tests.rs +++ b/src/mcp/federation_helpers_tests.rs @@ -1,13 +1,14 @@ //! Unit tests for the federation merge/parse/convert helpers. The //! FederationClient HTTP layer + resolve_group_targets are covered //! separately (federation/mod.rs and db_discovery/repos.rs respectively). -use super::{convert_remote_item, merge_ranked_lists}; +use super::{convert_remote_item, merge_ranked_lists, parse_search_items_from_call_result}; use crate::federation::RemoteSearchItem; use crate::mcp::types::SearchResultItem; +use rmcp::model::{CallToolResult, ContentBlock}; fn local_item(chunk_id: u32, score: f32) -> SearchResultItem { SearchResultItem { - chunk_id, + chunk_id: Some(chunk_id), path: format!("local/{chunk_id}.rs"), start_line: 1, end_line: 2, @@ -38,8 +39,8 @@ fn merge_interleaves_disjoint_lists_by_rank() { assert_eq!(merged.len(), 5); // Rank-0 of each list: score 1/(20+0+1) = 1/21 ≈ 0.0476 — both rank 0 // tiebreak on insertion order (local list first). - let top_ids: Vec = merged.iter().map(|i| i.chunk_id).collect(); - assert_eq!(top_ids, vec![1, 10, 2, 11, 3]); + let top_ids: Vec> = merged.iter().map(|i| i.chunk_id).collect(); + assert_eq!(top_ids, vec![Some(1), Some(10), Some(2), Some(11), Some(3)]); // Scores must be reassigned to the RRF value. assert!((merged[0].score - 1.0 / 21.0).abs() < 1e-6); } @@ -69,7 +70,7 @@ fn convert_tags_source_and_namespaced_chunk_ref() { let item = convert_remote_item("cloud", "inriver", remote); assert_eq!(item.source.as_deref(), Some("cloud/inriver")); assert_eq!(item.chunk_ref.as_deref(), Some("cloud/inriver:42")); - assert_eq!(item.chunk_id, 42); // local id preserved for rendering + assert_eq!(item.chunk_id, Some(42)); // local id preserved for rendering assert_eq!(item.path, "cloud/kb.md"); } @@ -92,6 +93,21 @@ fn convert_falls_back_to_snippet_as_content() { let item = convert_remote_item("peer", "someproj", remote); assert_eq!(item.content.as_deref(), Some("matched line")); assert!(item.chunk_ref.is_none(), "no chunk_ref without chunk_id"); + // The fabricated-id regression pin: a literal hit must carry NO chunk_id, + // not a default 0. `0` here would let a caller hand-build a bogus + // "/:0" ref that get_chunk resolves to an unrelated chunk. + assert_eq!( + item.chunk_id, None, + "literal hit must not render a chunk_id" + ); + // And the omission must be a true JSON omission, not an explicit null + // (serde_json::json! renders None as null; the struct's + // skip_serializing_if must drop the key entirely). + let json = serde_json::to_value(&item).unwrap(); + assert!( + json.get("chunk_id").is_none(), + "chunk_id key must be absent, got: {json}" + ); } #[test] @@ -126,3 +142,100 @@ fn parse_federated_chunk_ref_rejects_garbage() { assert!(super::parse_federated_chunk_ref("no-colon-here").is_none()); assert!(super::parse_federated_chunk_ref("cloud:notanumber").is_none()); } + +// === parse_search_items_from_call_result (serve-delegation re-parse) === + +fn call_result_with_json(json: &str) -> CallToolResult { + CallToolResult::success(vec![ContentBlock::text(json.to_string())]) +} + +#[test] +fn parse_literal_items_carry_no_chunk_id() { + // The production literal shape: LiteralSearchResponse items have no + // chunk_id at all. Re-parsing must keep the id absent — the fabricated + // `0` this test replaces once let callers build a bogus + // "/:0" chunk_ref that get_chunk resolved to an + // unrelated chunk. + let payload = serde_json::json!({ + "results": [ + { + "path": "custom-kb/troubleshoot/example.md", + "start_line": 3, + "end_line": 3, + "snippet": "tags: classification-importer", + "score": 0.42, + "kind": "Section" + } + ] + }) + .to_string(); + let items = parse_search_items_from_call_result(&call_result_with_json(&payload), "literal"); + assert_eq!(items.len(), 1); + assert_eq!(items[0].chunk_id, None, "literal hit must not gain an id"); + // Absent from the re-serialized JSON too — a null would look like a + // real (if invalid) id to a caller combining fields by hand. + let json = serde_json::to_value(&items[0]).unwrap(); + assert!( + json.get("chunk_id").is_none(), + "chunk_id key must be absent, got: {json}" + ); + // The snippet must still map into content (uniform merged rendering). + assert_eq!( + items[0].content.as_deref(), + Some("tags: classification-importer") + ); +} + +#[test] +fn parse_literal_item_preserves_an_explicit_chunk_id() { + // Defensive tolerance: if a peer ever includes a chunk_id in its + // literal payload, it is a real id and must survive the re-parse + // (Some), not be forced to None. + let payload = serde_json::json!({ + "results": [ + { "path": "a.md", "start_line": 1, "end_line": 1, "snippet": "hit", "score": 0.1, "chunk_id": 7 } + ] + }) + .to_string(); + let items = parse_search_items_from_call_result(&call_result_with_json(&payload), "literal"); + assert_eq!(items.len(), 1); + assert_eq!(items[0].chunk_id, Some(7)); +} + +#[test] +fn parse_semantic_items_keep_real_ids() { + let payload = serde_json::json!({ + "results": [ + { + "chunk_id": 321, + "path": "custom-kb/howto/example.md", + "start_line": 1, + "end_line": 12, + "kind": "Section", + "score": 0.9 + } + ] + }) + .to_string(); + let items = parse_search_items_from_call_result(&call_result_with_json(&payload), "semantic"); + assert_eq!(items.len(), 1); + assert_eq!(items[0].chunk_id, Some(321)); + assert_eq!(items[0].path, "custom-kb/howto/example.md"); +} + +#[test] +fn parse_unparseable_payload_yields_empty_list() { + // Documents the pre-existing degrade for malformed delegation JSON: + // empty list, not an error. Pinned as-is so a future tightening is a + // deliberate change, not an accident. + assert!(parse_search_items_from_call_result( + &call_result_with_json("not json at all"), + "literal" + ) + .is_empty()); + assert!(parse_search_items_from_call_result( + &call_result_with_json("{\"no_results_key\": true}"), + "literal" + ) + .is_empty()); +} diff --git a/src/mcp/find.rs b/src/mcp/find.rs new file mode 100644 index 00000000..673f656c --- /dev/null +++ b/src/mcp/find.rs @@ -0,0 +1,459 @@ +//! Consolidated `find` tool (definition/usages/imports/dependents dispatch) +//! plus its internals. Extracted from mod.rs (todo #105). The `#[tool]` +//! method registers through the per-module router that mod.rs merges inside +//! its own `merged_tool_router` composition point. + +use super::*; +use rmcp::{ + handler::server::wrapper::Parameters, + model::{CallToolResult, ContentBlock}, + tool, tool_router, ErrorData as McpError, +}; + +#[tool_router(router = find_router, vis = "pub(crate)")] +impl CodesearchService { + /// Unified symbol navigation — dispatches based on `kind`. + #[tool( + description = "Unified symbol navigation. Set `kind` to choose the action:\n\n- `definition` (default): locate where a symbol is defined (function, class, struct, etc.)\n- `usages`: find call-sites of a symbol via LEXICAL TEXT matching — hits may be docs/comments rather than code references; ranking puts source files first and a `note` field flags the precise upgrade path when one exists. On C#/TypeScript projects ALWAYS prefer `find_impact` for usages — it returns exact SCIP references, while this kind is only a text fallback\n- `imports`: list all imports/dependencies declared in a file (set `symbol` to the file path)\n- `dependents`: find all files that import or depend on a module, file, or symbol\n\nFor `imports`, set `symbol` to a file path. For other kinds, `symbol` is the symbol name.\n\nIMPORTANT (multi-repo): always specify either `project` (single repo) or `group` (cross-repo). Omitting both in multi-repo mode returns a `scope_required` error with the list of available projects and groups. If the user has not indicated which repository to search, ask them to choose." + )] + pub(crate) async fn find( + &self, + Parameters(request): Parameters, + ) -> Result { + let kind = request + .kind + .as_deref() + .unwrap_or("definition") + .to_lowercase(); + tracing::info!( + "📥 find(symbol={:?}, kind={}, project={:?}, group={:?})", + request.symbol, + kind, + request.project, + request.group, + ); + match kind.as_str() { + "definition" => { + let def_req = FindDefinitionRequest { + symbol: request.symbol, + kind: request.definition_kind, + limit: request.limit, + project: request.project, + group: request.group, + }; + self.find_definition(Parameters(def_req)).await + } + "usages" => { + let usages_req = FindUsagesRequest { + symbol: request.symbol, + limit: request.limit, + project: request.project, + group: request.group, + }; + self.find_usages(Parameters(usages_req)).await + } + "imports" => { + let imports_req = FindImportsRequest { + path: request.symbol, + project: request.project, + group: request.group, + }; + self.find_imports(Parameters(imports_req)).await + } + "dependents" => { + let dep_req = FindDependentsRequest { + symbol_or_path: request.symbol, + limit: request.limit, + project: request.project, + group: request.group, + }; + self.find_dependents(Parameters(dep_req)).await + } + _ => Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Unknown find kind '{}'. Use `definition`, `usages`, `imports`, or `dependents`.", + kind + ))])), + } + } + + // === find_definition internal === + + /// Internal: find symbol definitions, used by `find(kind="definition")`. + async fn find_definition( + &self, + Parameters(request): Parameters, + ) -> Result { + let limit = request.limit.unwrap_or(20); + + tracing::debug!( + "MCP find_definition: symbol='{}', kind={:?}, limit={}", + request.symbol, + request.kind, + limit + ); + + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // Stores that failed during this lookup. Without this, "the symbol may + // not be indexed" below is emitted as a confident diagnosis even when + // no store ever answered. + let mut find_warnings: Vec = Vec::new(); + + // FTS search — multi-store or single + let fts_results = if let Some(ref sv) = ctx.stores_vec { + let sa = ctx.store_aliases.as_ref().unwrap(); + self.with_fts_store_read_multi( + |fts_store| fts_store.search(&request.symbol, limit * 3, None), + sv.clone(), + sa, + ) + .await + .unwrap_or_default() + .into_results(&mut find_warnings, "definition search") + } else { + match self + .with_fts_store_read_for( + |fts_store| fts_store.search(&request.symbol, limit * 3, None), + ctx.stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching: {e:#}" + ))])); + } + } + }; + + if fts_results.is_empty() { + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result( + format!( + "No definition found for '{}'. The symbol may not be indexed.", + request.symbol + ), + &find_warnings, + ), + )])); + } + + // Resolve chunk metadata and filter by definition kinds + let requested_kind = request.kind.clone(); + let mut items: Vec = if let Some(ref sv) = ctx.stores_vec { + let aliases = ctx.aliases(); + let mut items: Vec = Vec::new(); + 'outer: for fts_result in &fts_results { + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + let looked_up = store.get_chunk(fts_result.chunk_id); + if let Err(ref e) = looked_up { + // `Ok(None)` = chunk not in this store (normal during + // fan-out); `Err` = broken store. Skipping the `Err` + // silently made a dead store look like "symbol not + // found" — carry it in the warnings channel instead. + note_store_failure( + &mut find_warnings, + aliases, + store_idx, + "chunk lookup", + e, + ); + } + if let Ok(Some(chunk)) = looked_up { + // Skip non-definition kinds — try next FTS result, not next store + if !DEFINITION_KINDS.contains(&chunk.kind.as_str()) { + continue 'outer; + } + if let Some(ref rk) = requested_kind { + if chunk.kind != *rk { + continue 'outer; + } + } + items.push(ReferenceItem { + chunk_id: fts_result.chunk_id, + path: chunk.path, + line: chunk.start_line, + kind: chunk.kind, + signature: chunk.signature, + score: fts_result.score, + }); + if items.len() >= limit { + break 'outer; + } + break; // Found in this store — move to next FTS result + } + } + // If we get here, the chunk was Ok(None) in every store (not + // held anywhere — skip it) or its lookups failed (noted in + // find_warnings above). + } + items + } else { + match self + .with_vector_store_read_for( + |store| { + // Resolve chunk metadata first: a store `Err` must + // reach the error arm below ("Error opening database") + // instead of masquerading as a non-definition or + // missing chunk — `Ok(None)` alone is a true miss. + let resolved: anyhow::Result> = fts_results + .iter() + .map(|fts_result| { + let chunk = store.get_chunk(fts_result.chunk_id)?; + Ok((chunk, fts_result.chunk_id, fts_result.score)) + }) + .collect(); + let items = resolved? + .into_iter() + .filter_map(|(looked_up, chunk_id, score)| { + let chunk = looked_up?; + if !DEFINITION_KINDS.contains(&chunk.kind.as_str()) { + return None; + } + if let Some(ref requested_kind) = requested_kind { + if chunk.kind != *requested_kind { + return None; + } + } + Some(ReferenceItem { + chunk_id, + path: chunk.path, + line: chunk.start_line, + kind: chunk.kind, + signature: chunk.signature, + score, + }) + }) + .take(limit) + .collect(); + Ok(items) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error opening database: {e:#}" + ))])); + } + } + }; + + // Prefix paths with alias for multi-repo identification + for item in &mut items { + item.path = ctx.prefix_result_path(&item.path); + } + + respond_with_items(&items, &find_warnings, || { + format!( + "No definition found for '{}'. Try find_usages() to find references, \ + or broaden your search.", + request.symbol + ) + }) + } + + // === find_usages tool === + + async fn find_usages( + &self, + Parameters(request): Parameters, + ) -> Result { + self.find_usages_impl( + request.symbol.clone(), + request.limit.unwrap_or(20), + request.project, + request.group, + ) + .await + } + + /// Shared implementation for find_usages (used by `find(kind="usages")`). + async fn find_usages_impl( + &self, + symbol: String, + limit: usize, + project: Option, + group: Option, + ) -> Result { + tracing::debug!("MCP find_usages: symbol='{}', limit={}", symbol, limit); + + // Resolve project/group routing + let ctx = match self.resolve_routing(&project, &group, false, "find").await { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // See `find_definition`: an empty result and a dead store must not + // produce the same sentence. + let mut find_warnings: Vec = Vec::new(); + + // FTS search — multi-store or single + let fts_results = if let Some(ref sv) = ctx.stores_vec { + let sa = ctx.store_aliases.as_ref().unwrap(); + self.with_fts_store_read_multi( + |fts_store| fts_store.search(&symbol, limit * 2, None), + sv.clone(), + sa, + ) + .await + .unwrap_or_default() + .into_results(&mut find_warnings, "usage search") + } else { + match self + .with_fts_store_read_for( + |fts_store| fts_store.search(&symbol, limit * 2, None), + ctx.stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching: {e:#}" + ))])); + } + } + }; + + if fts_results.is_empty() { + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result( + format!("No usages found for '{symbol}'. The symbol may not be indexed."), + &find_warnings, + ), + )])); + } + + // Resolve chunks and exclude definition chunks + let mut items: Vec = if let Some(ref sv) = ctx.stores_vec { + let aliases = ctx.aliases(); + let mut items: Vec = Vec::new(); + for fts_result in &fts_results { + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + let looked_up = store.get_chunk(fts_result.chunk_id); + if let Err(ref e) = looked_up { + // Same rule as find_definition: `Err` is a broken + // store, not "no usages" — carry it in the channel. + note_store_failure( + &mut find_warnings, + aliases, + store_idx, + "chunk lookup", + e, + ); + } + if let Ok(Some(chunk)) = looked_up { + if !is_definition_chunk(&chunk.kind, &chunk.signature, &symbol) { + items.push(ReferenceItem { + chunk_id: fts_result.chunk_id, + path: chunk.path, + line: chunk.start_line, + kind: chunk.kind, + signature: chunk.signature, + score: fts_result.score, + }); + } + break; + } + } + } + items + } else { + match self + .with_vector_store_read_for( + |store| { + // Resolve first: a store `Err` must reach the error + // arm below instead of masquerading as a definition + // chunk or a miss — `Ok(None)` alone is a true miss. + let resolved: anyhow::Result> = fts_results + .iter() + .map(|fts_result| { + let chunk = store.get_chunk(fts_result.chunk_id)?; + Ok((chunk, fts_result.chunk_id, fts_result.score)) + }) + .collect(); + let items = resolved? + .into_iter() + .filter_map(|(looked_up, chunk_id, score)| { + let chunk = looked_up?; + if is_definition_chunk(&chunk.kind, &chunk.signature, &symbol) { + return None; + } + Some(ReferenceItem { + chunk_id, + path: chunk.path, + line: chunk.start_line, + kind: chunk.kind, + signature: chunk.signature, + score, + }) + }) + .collect(); + Ok(items) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error opening database: {e:#}" + ))])); + } + } + }; + + // Prefix paths with alias for multi-repo identification + for item in &mut items { + item.path = ctx.prefix_result_path(&item.path); + } + + // Lexical FTS ranks docs, comments and code by the same text score, + // so a usages query can bury the real call-sites under markdown. The + // cut to `limit` MUST happen after the re-order: BM25 systematically + // scores short markdown blocks above long source files (document + // length normalisation), so ranking after a `take(limit)` would have + // nothing left to reorder exactly when it matters most. Stable sort + // preserves score order within each group; nothing is filtered. + rank_code_first(&mut items); + items.truncate(limit); + + // When the hits include SCIP-backed source files and a precise + // backend is installed, tell the agent the exact upgrade path — + // otherwise the lexical list silently stands in for real references. + let note = scip_usages_note(&self.symbol_registry, &items, &symbol); + + respond_with_items_noted(&items, &find_warnings, note.as_deref(), || { + format!( + "No usages found for '{symbol}' (only definitions were found). Try \ + find_definition() to locate the declaration." + ) + }) + } +} diff --git a/src/mcp/find_impact.rs b/src/mcp/find_impact.rs new file mode 100644 index 00000000..035a1957 --- /dev/null +++ b/src/mcp/find_impact.rs @@ -0,0 +1,577 @@ +use super::types::FindImpactRequest; +use super::CodesearchService; +use crate::symbols::SymbolReference; +use rmcp::{ + handler::server::wrapper::Parameters, + model::{CallToolResult, ContentBlock}, + tool, tool_router, ErrorData as McpError, +}; +use std::path::{Path, PathBuf}; + +/// Resolve the `find_impact` wall-clock budget: env var → +/// `DEFAULT_FIND_IMPACT_BUDGET_SECS`. `0` disables the budget (unbounded +/// lookup, the pre-budget behaviour). Mirrors +/// `resolve_proxy_idle_disconnect_secs`. +fn resolve_find_impact_budget_secs() -> u64 { + std::env::var(crate::constants::FIND_IMPACT_BUDGET_SECS_ENV) + .ok() + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(crate::constants::DEFAULT_FIND_IMPACT_BUDGET_SECS) +} + +/// Outcome of a budget-bounded `find_impact` lookup. +pub(crate) enum ImpactLookupOutcome { + /// The lookup finished within the budget (or the budget is disabled). + /// `Err` preserves the store/helper failure for the caller to report. + Done(Result, anyhow::Error>), + /// The budget overran; the lookup keeps running in the background. + Busy { + /// What is still running (goes into the busy envelope verbatim). + state: String, + /// Wall-clock time actually waited before giving up. + waited_ms: u64, + }, +} + +/// Race a `find_impact` lookup against its wall-clock budget. +/// +/// `lookup` is the already-offloaded lookup future (the handler runs the +/// blocking SCIP call on `spawn_blocking`); it is NOT cancelled on overrun — +/// dropping the future abandons it while the detached blocking task keeps +/// running, so its reference-cache writes still land in LMDB and the retry +/// hinted by the busy answer is served warm. `budget_secs == 0` disables the +/// race entirely. Kept generic over the future so tests can plant a sleeping +/// handler instead of a real SCIP helper. +pub(crate) async fn find_impact_with_budget( + budget_secs: u64, + state: String, + lookup: F, +) -> ImpactLookupOutcome +where + F: std::future::Future, anyhow::Error>>, +{ + if budget_secs == 0 { + return ImpactLookupOutcome::Done(lookup.await); + } + let started = std::time::Instant::now(); + match tokio::time::timeout(std::time::Duration::from_secs(budget_secs), lookup).await { + Ok(result) => ImpactLookupOutcome::Done(result), + Err(_elapsed) => ImpactLookupOutcome::Busy { + state, + waited_ms: u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + }, + } +} + +/// Collapse exact-duplicate references from the SCIP find-refs output. +/// +/// The helper can emit multiple occurrences of the same symbol at the same +/// file:line (declaration plus multiple roles on one line), which reaches the +/// agent as visible noise (observed live: a definition listed 5×). Two +/// references that are identical in ALL of (file, line range, kind) carry no +/// information the agent could act on separately — `SymbolReference` has no +/// column, so two genuinely distinct same-line calls are indistinguishable +/// from duplicates and collapse too, which is the right call for a caller +/// that wants "where is this used", not occurrence counts. Order is stable. +fn dedupe_references( + refs: Vec, +) -> Vec { + let mut seen: std::collections::HashSet<(String, u32, u32, String)> = + std::collections::HashSet::with_capacity(refs.len()); + let mut out = Vec::with_capacity(refs.len()); + for r in refs { + let key = ( + r.file.to_string_lossy().into_owned(), + r.start_line, + r.end_line, + r.kind.clone(), + ); + if seen.insert(key) { + out.push(r); + } + } + out +} + +#[tool_router(router = find_impact_router, vis = "pub(crate)")] +impl CodesearchService { + /// Symbol impact analysis — returns transitive call-sites of a symbol with file/line precision. + /// + /// The recommended tool for "who calls X?" / "what breaks if I rename X?". Uses + /// language-specific semantic analysis (SCIP) to find all references, enabling agents + /// to plan refactors with IDE-class accuracy instead of text-matching grep heuristics. + /// Precision backends ship per language: C# (bundled `scip-csharp` helper, + /// `-with-csharp` releases) and TypeScript (`scip-typescript`, resolved via `npx` + /// or `CODESEARCH_SCIP_TYPESCRIPT`). If no backend is installed for the target + /// language, the response reports it — fall back to `find` with `kind="usages"` + /// (lexical) only then. + /// + /// Ambiguity contract: when several stored symbols match a name or a position + /// (overloads, same-line definitions), the answer is a typed `{"ambiguous": true, + /// "candidates": [...]}` envelope — the server never silently picks one. Re-call + /// with `symbol_key` set to exactly one candidate. Every resolved answer names the + /// selected canonical key in `resolved_symbol`. + #[tool( + description = "Symbol impact analysis — find all references to a symbol with IDE-class precision (SCIP).\n\nThe right tool for \"who calls X?\" / \"what breaks if I rename X?\". Returns transitive call-sites with file/line precision, enabling agents to plan refactors without missing a caller. More accurate than text-based `find kind=\"usages\"` because it understands language semantics.\n\nInput variants (mutually exclusive):\n- By name: `{ \"symbol_name\": \"FieldDefinition.Validate\", \"project\": \"myrepo\" }`\n- By position: `{ \"file\": \"src/Validation/FieldDefinition.cs\", \"line\": 42, \"project\": \"myrepo\" }`\n- By exact canonical key: `{ \"symbol_key\": \"csharp . . . FieldDefinition#Validate().\", \"project\": \"myrepo\" }`\n\nAMBIGUITY: if several stored symbols match (overloads, same-line definitions), the answer is `{\"ambiguous\": true, \"query\": ..., \"candidates\": [...]}` — pick one candidate and re-call with `symbol_key` set to it verbatim. A resolved answer names the selected canonical key in `resolved_symbol`; never assume which overload answered.\n\nWARNINGS: a resolved answer may carry a `\"warnings\": [\"...\"]` array — non-empty means the reference list may be INCOMPLETE because a helper failure was survived rather than fatal (a project that failed to compile, an exception during reference resolution, a non-zero scip-typescript exit). Each entry names what failed. Treat warnings as a prompt to reindex the project (or cross-check with `find` `kind=\"usages\"`) before concluding \"no callers\" — absent or empty means the answer is as complete as the index knows.\n\nLANGUAGE: if `language` is omitted, position lookups auto-detect it from the file extension; with several SCIP helpers installed the answer asks you to name one — pass `language` to avoid the round-trip.\n\nPrecision backends (SCIP) ship per language; C# (bundled `scip-csharp` helper, `-with-csharp` releases) and TypeScript (via `npx` or `CODESEARCH_SCIP_TYPESCRIPT`) are available today. For Rust/Python/Go/etc., use `find` with `kind=\"usages\"` as a text-based fallback until SCIP backends for those languages ship.\n\nOn a busy answer (`\"busy\": true`): sleep `retry_after_seconds` and retry the SAME call. Busy is progress, not failure — never fall back to text search on busy.\n\nIMPORTANT (multi-repo): always specify `project` (single repo). Omitting `project` in multi-repo mode returns a `scope_required` error." + )] + async fn find_impact( + &self, + Parameters(request): Parameters, + ) -> Result { + tracing::info!( + "📥 find_impact(symbol_name={:?}, file={:?}, line={:?}, language={:?}, project={:?})", + request.symbol_name, + request.file, + request.line, + request.language, + request.project, + ); + + // Validate input: exactly one of symbol_key / symbol_name / file+line. + let has_name = request + .symbol_name + .as_ref() + .is_some_and(|s| !s.trim().is_empty()); + let has_position = request.file.is_some() && request.line.is_some(); + let has_key = request + .symbol_key + .as_ref() + .is_some_and(|s| !s.trim().is_empty()); + if !has_name && !has_position && !has_key { + return Ok(CallToolResult::success(vec![ContentBlock::text( + "Must provide `symbol_name`, both `file` and `line`, or an exact `symbol_key`." + .to_string(), + )])); + } + // An explicit key IS the selection; combining it with a fuzzy query + // would let silent precedence decide the answer — the thing this + // contract exists to remove. Reject instead. + if has_key && (has_name || has_position) { + return Ok(CallToolResult::success(vec![ContentBlock::text( + "`symbol_key` is mutually exclusive with `symbol_name` and `file`+`line`: pass only the explicit selection.".to_string(), + )])); + } + + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find_impact") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + // Determine project root and db_path for the symbol index + let (project_root, db_path) = if let Some(ref alias) = ctx.project_alias { + let root = ctx + .alias_roots + .get(alias) + .map(PathBuf::from) + .unwrap_or_else(|| self.project_path.clone()); + // The symbol index DB lives alongside the vector DB + let db = root.join(crate::constants::DB_DIR_NAME); + (root, db) + } else { + // Single-repo / stdio mode: use the service's own paths + (self.project_path.clone(), self.db_path.clone()) + }; + + // Use the shared symbol indexer registry + let registry = &self.symbol_registry; + + // Determine which language to use + let language = request.language.clone().or_else(|| { + // Auto-detect from file extension + request.file.as_ref().and_then(|f| { + let ext = Path::new(f).extension()?.to_str()?.to_lowercase(); + match ext.as_str() { + "cs" => Some(crate::constants::LANG_CSHARP.to_string()), + "ts" | "tsx" | "mts" | "cts" => { + Some(crate::constants::LANG_TYPESCRIPT.to_string()) + } + _ => None, + } + }) + }); + + let indexer: &dyn crate::symbols::SymbolIndexer = match language { + Some(ref lang) => match registry.get(lang) { + Some(i) => i, + None => { + let available = registry.available_languages(); + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "No symbol indexer for language '{}'. Available languages: {:?}", + lang, available + ))])); + } + }, + None => { + // No language given and none detectable from a file path. + let installed = registry.installed_languages(); + if installed.is_empty() { + return Ok(CallToolResult::success(vec![ContentBlock::text( + "No symbol indexers installed. Install the `scip-csharp` helper for C# support, or `scip-typescript` (via npx) for TypeScript support.".to_string(), + )])); + } + if installed.len() > 1 { + // Several helpers installed: answering from one silently is + // the same silent pick the ambiguity contract removes. Ask. + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Several symbol indexes are installed ({}). Pass `language` (e.g. \"csharp\") so the lookup cannot silently answer from the wrong one.", + installed.join(", ") + ))])); + } + // Exactly one installed: the pick is deterministic. + match registry.get(&installed[0]) { + Some(i) => i, + None => { + unreachable!("installed_languages() returned a language with no indexer") + } + } + } + }; + + // Check if the helper is available + if !indexer.is_available() { + let error = crate::symbols::SymbolIndexError { + error: format!( + "Symbol indexer for '{}' is not available. The helper binary is not installed.", + indexer.language() + ), + available_languages: registry.available_languages(), + hint_for_agent: format!( + "Install the `-with-csharp` release variant, or set {} to the helper path.", + crate::constants::SCIP_CSHARP_HELPER_ENV + ), + }; + return Ok(CallToolResult::success(vec![ContentBlock::text( + serde_json::to_string(&error).unwrap_or_else(|_| error.error.clone()), + )])); + } + + // Perform the lookup under an internal wall-clock budget. + // + // `find_references_for_key` may invoke `scip-csharp find-refs` on a cache miss + // (lazy Opt-2 reference resolution). That subprocess can take several minutes + // on a large solution. The call therefore runs on `spawn_blocking` (it never + // blocks an async worker thread) and is raced against + // CODESEARCH_FIND_IMPACT_BUDGET_SECS: on overrun the caller gets a structured + // busy answer instead of the MCP client winning the timeout race. The + // blocking task is abandoned, not cancelled — its cache writes still land + // in LMDB, so the hinted retry is served warm; the lookup tracker + // (find_impact_tracker) makes that retry observe progress or the warm + // result explicitly instead of re-running the helper. + // Build the typed query. The validation above guarantees exactly + // one input form, so there is no precedence to guess at. + let query = if has_key { + crate::symbols::ImpactQuery::ExactKey( + request.symbol_key.clone().expect("has_key checked"), + ) + } else if has_name { + crate::symbols::ImpactQuery::Name( + request.symbol_name.clone().expect("has_name checked"), + ) + } else { + crate::symbols::ImpactQuery::Position { + file: self.normalize_symbol_query_path( + &project_root, + Path::new(request.file.as_ref().expect("has_position checked")), + ), + line: request.line.expect("has_position checked"), + } + }; + let what = match &query { + crate::symbols::ImpactQuery::Name(n) => format!("'{n}'"), + crate::symbols::ImpactQuery::ExactKey(k) => k.clone(), + crate::symbols::ImpactQuery::Position { file, line } => { + format!("{}:{}", file.display(), line) + } + }; + // The echo the response's `symbol` field has always carried — the + // query as asked. The selected identity travels in `resolved_symbol` + // from here on; the echo never claimed to be one. + let echo = what + .trim_start_matches('\'') + .trim_end_matches('\'') + .to_string(); + + let language_for_lookup = indexer.language().to_string(); + let busy_state = format!( + "resolving {} via the {} SCIP helper (cold reference cache)", + what, language_for_lookup + ); + let budget_secs = resolve_find_impact_budget_secs(); + + // Index fingerprint: the repository HEAD at response time. The + // non-fatal git read is offloaded like every blocking call; a + // failed read simply omits the field. Drift against + // `index_head_sha` is surfaced, never auto-reindexed (deliberate: + // reindexing a large solution on every branch switch would thrash). + let head_root = project_root.clone(); + let current_head_sha = + tokio::task::spawn_blocking(move || crate::symbols::current_git_head(&head_root)) + .await + .unwrap_or(None); + + // Shared result construction: the warm-retry path (below) must be + // byte-identical to a budget-fast completion, so both build the + // response through this one closure. + let build_impact = |references: Vec, + resolved_symbol: Option, + warnings: Vec| + -> crate::symbols::FindImpactResult { + crate::symbols::FindImpactResult { + symbol: echo.clone(), + resolved_symbol, + references: dedupe_references(references), + warnings, + index_age_seconds: indexer.index_age(&db_path), + language: indexer.language().to_string(), + scope: ctx + .project_alias + .map(|a| format!("project:{}", a)) + .unwrap_or_else(|| "local".to_string()), + index_head_sha: indexer.index_head_sha(&db_path), + current_head_sha: current_head_sha.clone(), + } + }; + + // Resolution: plain LMDB reads, run off the async runtime like every + // blocking call. Ambiguity surfaces HERE — before any expensive + // helper invocation — as a typed candidates answer; the server never + // silently picks among the query's matches (the old behaviour took + // the shortest fuzzy candidate and answered about the wrong symbol). + let registry_for_resolve = self.symbol_registry.clone(); + let language_for_resolve = language_for_lookup.clone(); + let db_path_for_resolve = db_path.clone(); + let query_for_resolve = query.clone(); + let resolution = match tokio::task::spawn_blocking(move || { + let indexer = registry_for_resolve + .get(&language_for_resolve) + .ok_or_else(|| { + anyhow::anyhow!( + "symbol indexer for '{}' disappeared mid-request", + language_for_resolve + ) + })?; + indexer.resolve_query(&db_path_for_resolve, &query_for_resolve) + }) + .await + { + Ok(inner) => inner, + // The handler returns an MCP error type, not anyhow::Error, so + // a JoinError is folded into the lookup-failure path below + // (classified stale/failed by the index age) instead of `?`. + Err(e) => Err(anyhow::anyhow!("symbol resolve task failed: {e:#}")), + }; + + let canonical = match resolution { + Err(e) => { + let failure = crate::symbols::SymbolLookupFailure::classify( + format!("{e:#}"), + indexer.index_age(&db_path), + ); + let json = + serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Ok(crate::symbols::KeyMatch::Ambiguous(candidates)) => { + let ambiguity = crate::symbols::SymbolAmbiguity { + ambiguous: true, + query: what, + candidates, + hint_for_agent: "Several stored symbols match this query. Re-call find_impact with `symbol_key` set to exactly one of `candidates`, verbatim.".to_string(), + }; + let json = serde_json::to_string(&ambiguity) + .unwrap_or_else(|_| "{\"ambiguous\":true}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Ok(crate::symbols::KeyMatch::NotFound) if has_key => { + // An explicit key that misses is a loud failure, not an + // empty answer: the caller selected that key deliberately, + // so a miss means the answer it came from and the index + // have drifted apart. + let failure = crate::symbols::SymbolLookupFailure { + error: format!( + "symbol_key '{}' is not in the {} index", + echo, language_for_lookup + ), + class: crate::symbols::SymbolLookupFailureClass::Failed, + hint_for_agent: "The index was likely rebuilt since the ambiguous answer. Re-run the original name or position query to list the current candidates instead of retrying the key." + .to_string(), + }; + let json = + serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Ok(crate::symbols::KeyMatch::NotFound) => { + // Preserve the historical contract for fuzzy queries: an + // unresolvable name/position answers empty references. + let impact = build_impact(Vec::new(), None, Vec::new()); + let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Ok(crate::symbols::KeyMatch::Resolved(canonical)) => canonical, + }; + + // Background continuation: consult the tracker before starting a + // (potentially cold, minutes-long) lookup. A retry of an overran + // lookup observes progress or the warm result instead of racing a + // second identical helper subprocess against the cold cache. + let tracker_key: find_impact_tracker::LookupKey = (db_path.clone(), what.clone()); + match find_impact_tracker::IMPACT_LOOKUP_TRACKER.check(&tracker_key) { + Some(find_impact_tracker::TrackedStatus::Running { elapsed_ms }) => { + tracing::info!( + "find_impact retry: lookup still running ({}ms elapsed): {}", + elapsed_ms, + busy_state + ); + let busy = crate::symbols::SymbolLookupBusy { + busy: true, + state: busy_state.clone(), + waited_ms: elapsed_ms, + advice: format!( + "still running ({}s elapsed); retry the same call in ~{}s", + elapsed_ms / 1000, + budget_secs.max(1) + ), + retry_after_seconds: budget_secs.max(1), + }; + let json = + serde_json::to_string(&busy).unwrap_or_else(|_| "{\"busy\":true}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Some(find_impact_tracker::TrackedStatus::Done(Ok(references))) => { + tracing::info!( + "find_impact retry: serving warm result ({} references) from the finished background lookup: {}", + references.len(), + busy_state + ); + // The warm retry must carry the same honesty as a fresh + // answer: read the persisted warnings for this key. + let warnings = indexer.lookup_warnings(&db_path, &canonical); + let impact = build_impact(references, Some(canonical.clone()), warnings); + let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + Some(find_impact_tracker::TrackedStatus::Done(Err(chain))) => { + // Same classification as a fresh failure: the tracked chain + // is already `{:#}`-rendered, the index age decides the class. + let failure = crate::symbols::SymbolLookupFailure::classify( + chain, + indexer.index_age(&db_path), + ); + let json = + serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + None => {} + } + + let registry_for_lookup = self.symbol_registry.clone(); + let db_path_for_lookup = db_path.clone(); + let canonical_for_lookup = canonical.clone(); + let lookup_entry = find_impact_tracker::IMPACT_LOOKUP_TRACKER.register(tracker_key.clone()); + let lookup_entry_in_task = lookup_entry; + let lookup = async move { + tokio::task::spawn_blocking(move || { + let indexer = registry_for_lookup + .get(&language_for_lookup) + .ok_or_else(|| { + anyhow::anyhow!( + "symbol indexer for '{}' disappeared mid-request", + language_for_lookup + ) + })?; + let result = + indexer.find_references_for_key(&db_path_for_lookup, &canonical_for_lookup); + // Record INSIDE the blocking task: the handler's awaiting + // future is dropped at budget overrun, but this detached + // task survives and the recorded outcome is what the hinted + // retry observes. (`anyhow::Error` is not `Clone`, so the + // failure side is recorded as its rendered `{:#}` chain.) + let recorded = result.as_ref().map_err(|e| format!("{e:#}")).cloned(); + lookup_entry_in_task.finish(recorded); + result + }) + .await + .map_err(|e| anyhow::anyhow!("symbol lookup task failed: {e:#}"))? + }; + + match find_impact_with_budget(budget_secs, busy_state, lookup).await { + ImpactLookupOutcome::Done(Ok(references)) => { + // Completed within the budget: nothing is in flight, so a + // later lookup must consult the real cache, not the tracker. + find_impact_tracker::IMPACT_LOOKUP_TRACKER.remove(&tracker_key); + // Surface what the lookup survived: a partial answer must + // say so in its own payload, not only in a log line. + let warnings = indexer.lookup_warnings(&db_path, &canonical); + let impact = build_impact(references, Some(canonical), warnings); + let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } + ImpactLookupOutcome::Done(Err(e)) => { + find_impact_tracker::IMPACT_LOOKUP_TRACKER.remove(&tracker_key); + // Typed failure envelope (busy/stale/failed must stay + // machine-branchable; the index age decides stale vs failed). + let failure = crate::symbols::SymbolLookupFailure::classify( + format!("{e:#}"), + indexer.index_age(&db_path), + ); + let json = + serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } + ImpactLookupOutcome::Busy { state, waited_ms } => { + tracing::warn!( + "find_impact budget overrun after {}ms (budget {}s): {} — answering busy, lookup continues in background", + waited_ms, + budget_secs, + state + ); + let busy = crate::symbols::SymbolLookupBusy { + busy: true, + state, + waited_ms, + advice: format!( + "retry the same call in ~{}s; the lookup keeps running in the background and the retry is served from cache once it completes", + budget_secs.max(1) + ), + retry_after_seconds: budget_secs.max(1), + }; + let json = + serde_json::to_string(&busy).unwrap_or_else(|_| "{\"busy\":true}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } + } + } +} + +#[cfg(test)] +#[path = "find_impact_tests.rs"] +mod find_impact_tests; + +// `#[path]` is load-bearing: a plain `mod` from find_impact.rs would resolve +// under src/mcp/find_impact/, and the file stays at src/mcp/. +#[path = "find_impact_tracker.rs"] +mod find_impact_tracker; + +// ── REST mirror ── +// Defined here rather than beside the other rest_* handlers in the parent +// module because it calls the #[tool] method above, which is private to +// this module; the parent would need its visibility widened instead. + +/// REST mirror of the `find_impact` MCP tool: POST a `FindImpactRequest` +/// body, receive the tool's JSON payload. Read-only, same auth class as +/// the other REST mirrors (see `FIND_IMPACT_PATH`). +pub(crate) async fn rest_find_impact_handler( + axum::extract::State(state): axum::extract::State>, + axum::Json(req): axum::Json, +) -> Result, super::RestError> { + let service = super::make_service(&state)?; + let result = service + .find_impact(Parameters(req)) + .await + .map_err(super::mcp_err_to_http)?; + Ok(axum::Json(super::call_tool_result_to_json(result))) +} diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs new file mode 100644 index 00000000..a07b7927 --- /dev/null +++ b/src/mcp/find_impact_tests.rs @@ -0,0 +1,813 @@ +//! Tests for the `find_impact` wall-clock budget. +//! +//! `find_impact_with_budget` is generic over the lookup future, so these +//! tests plant a *sleeping handler* (a future that sleeps past the budget, +//! the same busy-serve simulation the proxy tests use) instead of a real +//! SCIP helper — the budget race is exercised in milliseconds. The env +//! resolution is tested separately against `resolve_find_impact_budget_secs` +//! with `#[serial]` + `EnvRestore` per the repo rule for env mutation. + +use super::{find_impact_with_budget, resolve_find_impact_budget_secs, ImpactLookupOutcome}; +use crate::constants::{DEFAULT_FIND_IMPACT_BUDGET_SECS, FIND_IMPACT_BUDGET_SECS_ENV}; +use crate::symbols::{SymbolLookupBusy, SymbolReference}; +use std::path::PathBuf; +use std::time::Duration; + +/// A lookup future that sleeps then succeeds — the planted busy handler. +async fn sleeping_lookup(delay: Duration) -> anyhow::Result> { + tokio::time::sleep(delay).await; + Ok(vec![SymbolReference { + file: PathBuf::from("src/x.rs"), + start_line: 1, + end_line: 2, + kind: "definition".to_string(), + }]) +} + +fn sample_state() -> String { + "resolving 'Ns.I.M' via the csharp SCIP helper".to_string() +} + +#[tokio::test] +async fn budget_overrun_returns_busy_with_wait_time() { + // Handler sleeps 2s, budget is 1s → the race must fire busy at ~1s, + // well before the handler completes. + let outcome = find_impact_with_budget( + 1, + sample_state(), + sleeping_lookup(Duration::from_millis(2_000)), + ) + .await; + match outcome { + ImpactLookupOutcome::Busy { state, waited_ms } => { + assert_eq!(state, sample_state()); + assert!( + (900..=2_000).contains(&waited_ms), + "busy must fire at ~the 1s budget, waited_ms={waited_ms}" + ); + } + ImpactLookupOutcome::Done(_) => panic!("a 2s handler must overrun a 1s budget"), + } +} + +#[tokio::test] +async fn fast_lookup_completes_within_budget_passes_through() { + let outcome = find_impact_with_budget( + 60, + sample_state(), + sleeping_lookup(Duration::from_millis(10)), + ) + .await; + match outcome { + ImpactLookupOutcome::Done(Ok(refs)) => { + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].file, PathBuf::from("src/x.rs")); + assert_eq!(refs[0].kind, "definition"); + } + _ => panic!("a 10ms handler must complete inside a 60s budget"), + } +} + +#[tokio::test] +async fn lookup_failure_passes_through_with_error_chain() { + let outcome = find_impact_with_budget(60, sample_state(), async { + tokio::time::sleep(Duration::from_millis(5)).await; + Err(anyhow::anyhow!("helper exited 1").context("scip-csharp find-refs failed")) + }) + .await; + match outcome { + // The soft-string failure render is Step 3 scope; here we only pin + // that Done(Err) — not busy, not swallowed — reaches the handler. + ImpactLookupOutcome::Done(Err(e)) => { + let rendered = format!("{e:#}"); + assert!( + rendered.contains("scip-csharp find-refs failed") + && rendered.contains("helper exited 1"), + "error chain must survive: {rendered}" + ); + } + _ => panic!("a failing handler must surface as Done(Err)"), + } +} + +#[tokio::test] +async fn zero_budget_disables_the_race() { + // 0 disables the budget (repo-wide convention), so even a slow handler + // completes instead of being answered busy. + let outcome = find_impact_with_budget( + 0, + sample_state(), + sleeping_lookup(Duration::from_millis(50)), + ) + .await; + assert!(matches!(outcome, ImpactLookupOutcome::Done(Ok(_)))); +} + +#[test] +fn busy_envelope_serializes_the_five_documented_fields() { + let busy = SymbolLookupBusy { + busy: true, + state: "resolving 'Ns.I.M' via the csharp SCIP helper".to_string(), + waited_ms: 60_012, + advice: "retry the same call in ~60s".to_string(), + retry_after_seconds: 45, + }; + let json: serde_json::Value = serde_json::to_value(&busy).unwrap(); + assert_eq!(json["busy"], serde_json::Value::Bool(true)); + assert!(json["state"].is_string()); + assert_eq!(json["waited_ms"], serde_json::Value::from(60_012)); + assert_eq!( + json["retry_after_seconds"], + serde_json::Value::from(45), + "retry_after_seconds must be present: harnesses branch on it instead of parsing prose" + ); + let advice = json["advice"].as_str().unwrap(); + assert!( + advice.contains("retry the same call in ~"), + "advice must carry the retry hint: {advice}" + ); + // Exactly the documented envelope shape — the five fields, no extras. + // (Key ORDER is deliberately not asserted: serde_json::to_value routes + // through a BTreeMap and re-sorts keys, so an order assertion here would + // test serde's map type, not the handler. Field order on the wire comes + // from struct serialization and is irrelevant to JSON consumers.) + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec![ + "advice", + "busy", + "retry_after_seconds", + "state", + "waited_ms" + ] + ); +} + +#[test] +#[serial_test::serial] +fn budget_env_overrides_parses_and_falls_back() { + // Table-driven: (raw env value, expected seconds). Garbage and empty + // fall back to the default rather than erroring or clamping to 0 — + // 0 is a meaningful value ("disable"), so it must only ever come from + // an explicit "0". + let cases: &[(&str, u64)] = &[ + ("90", 90), + ("0", 0), + (" 45 ", 45), + ("junk", DEFAULT_FIND_IMPACT_BUDGET_SECS), + ("", DEFAULT_FIND_IMPACT_BUDGET_SECS), + ("-5", DEFAULT_FIND_IMPACT_BUDGET_SECS), + ]; + for (raw, expected) in cases { + let _guard = crate::testing::EnvRestore::set(&[(FIND_IMPACT_BUDGET_SECS_ENV, raw)]); + assert_eq!( + resolve_find_impact_budget_secs(), + *expected, + "env value {raw:?} must resolve to {expected}" + ); + } + // Absent → documented default. + let _guard = crate::testing::EnvRestore::remove(&[FIND_IMPACT_BUDGET_SECS_ENV]); + assert_eq!( + resolve_find_impact_budget_secs(), + DEFAULT_FIND_IMPACT_BUDGET_SECS + ); +} + +#[test] +fn failure_envelope_serializes_exactly_the_three_documented_fields() { + let failure = crate::symbols::SymbolLookupFailure::failed("helper exited 1"); + let json: serde_json::Value = serde_json::to_value(&failure).unwrap(); + assert_eq!(json["class"], "failed"); + assert_eq!(json["error"], "helper exited 1"); + let hint = json["hint_for_agent"].as_str().unwrap(); + assert!(!hint.is_empty(), "hint must be non-empty: {hint}"); + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!(keys, vec!["class", "error", "hint_for_agent"]); +} + +#[test] +fn classify_maps_unknown_index_age_to_stale_and_readability_to_failed() { + use crate::symbols::SymbolLookupFailureClass as Class; + // (index_age_seconds, expected class) — u64::MAX is what `index_age` + // returns whenever the index cannot be opened or read. + let cases: &[(u64, Class)] = &[ + (u64::MAX, Class::Stale), + (0, Class::Failed), + (3_600, Class::Failed), + ]; + for (age, expected) in cases { + let failure = crate::symbols::SymbolLookupFailure::classify("chain", *age); + assert_eq!(failure.class, *expected, "age {age} must classify"); + assert!(!failure.error.is_empty()); + } +} + +#[test] +fn failure_hints_are_actionable_per_class() { + let failed = crate::symbols::SymbolLookupFailure::failed("boom"); + let stale = crate::symbols::SymbolLookupFailure::stale("gone"); + assert!( + failed.hint_for_agent.contains("usages"), + "failed hint must point at the text-search fallback: {}", + failed.hint_for_agent + ); + assert!( + stale.hint_for_agent.contains("index"), + "stale hint must point at (re)building the index: {}", + stale.hint_for_agent + ); +} + +#[test] +fn fingerprint_fields_present_when_set_and_omitted_when_none() { + use crate::symbols::SymbolReference; + let base = |resolved_symbol: Option, + index_head_sha: Option, + current_head_sha: Option| { + crate::symbols::FindImpactResult { + symbol: "FieldDefinition.Validate".to_string(), + resolved_symbol, + references: vec![SymbolReference { + file: PathBuf::from("a.cs"), + start_line: 1, + end_line: 1, + kind: "definition".to_string(), + }], + warnings: Vec::new(), + index_age_seconds: 12, + language: "csharp".to_string(), + scope: "project:p".to_string(), + index_head_sha, + current_head_sha, + } + }; + let all: serde_json::Value = serde_json::to_value(base( + Some("csharp Ns . FieldDefinition#Validate().".to_string()), + Some("a".repeat(40)), + Some("b".repeat(40)), + )) + .unwrap(); + assert_eq!(all["index_head_sha"], "a".repeat(40)); + assert_eq!(all["current_head_sha"], "b".repeat(40)); + assert_eq!( + all["resolved_symbol"], "csharp Ns . FieldDefinition#Validate().", + "a resolved answer must name the selected canonical identity" + ); + + // None must OMIT the keys, not serialize null — old consumers see an + // unchanged shape when the identity/fingerprint is unknown (ambiguous + // and not-found answers travel in their own envelopes). + let neither: serde_json::Value = serde_json::to_value(base(None, None, None)).unwrap(); + let keys: Vec<&str> = neither + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert!(!keys.contains(&"index_head_sha"), "keys: {keys:?}"); + assert!(!keys.contains(&"current_head_sha"), "keys: {keys:?}"); + assert!(!keys.contains(&"resolved_symbol"), "keys: {keys:?}"); +} + +#[test] +fn ambiguity_envelope_serializes_the_four_documented_fields() { + let ambiguity = crate::symbols::SymbolAmbiguity { + ambiguous: true, + query: "'Validate'".to_string(), + candidates: vec![ + "csharp Ns . V#Validate().".to_string(), + "csharp Ns . V#Validate(System.String).".to_string(), + ], + hint_for_agent: "re-call with symbol_key".to_string(), + }; + let json: serde_json::Value = serde_json::to_value(&ambiguity).unwrap(); + assert_eq!(json["ambiguous"], serde_json::Value::Bool(true)); + assert_eq!(json["query"], "'Validate'"); + assert_eq!(json["candidates"].as_array().unwrap().len(), 2); + assert!(!json["hint_for_agent"].as_str().unwrap().is_empty()); + // Exactly the documented envelope shape — the four fields, no extras. + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec!["ambiguous", "candidates", "hint_for_agent", "query"] + ); +} + +// ── Handler-arm tests: the resolution contract through the tool method ── +// +// The adapter layer pins KeyMatch semantics; these pin the arms a caller +// actually sees: Ambiguous → typed candidates envelope, explicit-key miss +// → loud failure (never the empty answer), fuzzy NotFound → the +// historical empty-references contract, and the symbol_key combination +// rejection. All early-return before the budget race, so no busy paths +// are in play. + +use super::CodesearchService; +use crate::mcp::types::FindImpactRequest; +use rmcp::handler::server::wrapper::Parameters; + +async fn tool_text(service: &CodesearchService, req: FindImpactRequest) -> String { + let result = service + .find_impact(Parameters(req)) + .await + .expect("tool result"); + result + .content + .first() + .and_then(|c| c.as_text()) + .expect("text content") + .text + .clone() +} + +fn find_impact_request() -> FindImpactRequest { + FindImpactRequest { + symbol_name: None, + file: None, + line: None, + symbol_key: None, + language: Some("csharp".to_string()), + project: None, + group: None, + } +} + +/// A service whose db is a valid-but-empty index at `/.codesearch.db`. +fn build_service() -> (CodesearchService, tempfile::TempDir) { + let root = tempfile::tempdir().expect("tempdir"); + let db = root.path().join(".codesearch.db"); + std::fs::create_dir_all(&db).expect("db dir"); + std::fs::write( + db.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .expect("metadata"); + let stores = + std::sync::Arc::new(crate::index::SharedStores::new(&db, 2).expect("shared stores")); + let service = CodesearchService::new_with_stores(Some(root.path().to_path_buf()), Some(stores)) + .expect("service"); + (service, root) +} + +/// Two `Validate` overloads in scip_symbols + the simple-name index — +/// enough for every resolution arm (only key-presence is read). +fn populate_overload_fixture(db_path: &std::path::Path) { + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + let v1 = "csharp Ns . V#Validate().".to_string(); + let v2 = "csharp Ns . V#Validate(System.String).".to_string(); + let symbols: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_SYMBOLS_DB_NAME)) + .unwrap() + .unwrap(); + for key in [&v1, &v2] { + symbols.put(&mut wtxn, key.as_str(), &[1u8]).unwrap(); + } + let names: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_SIMPLE_NAMES_DB_NAME)) + .unwrap() + .unwrap(); + let mut payload = vec![1u8]; + payload.extend_from_slice(&bincode::serialize(&vec![v1.clone(), v2.clone()]).unwrap()); + names.put(&mut wtxn, "Validate", &payload).unwrap(); + wtxn.commit().unwrap(); +} + +/// The C# indexer must pass the `is_available` gate deterministically: +/// a dummy file with the expected helper filename, wired via the env +/// override (serialised — env mutation, per the repo rule). +fn make_helper_available(root: &tempfile::TempDir) -> crate::testing::EnvRestore { + let helper = root.path().join(if cfg!(windows) { + "scip-csharp.exe" + } else { + "scip-csharp" + }); + std::fs::write(&helper, b"dummy").expect("dummy helper file"); + crate::testing::EnvRestore::set(&[( + crate::constants::SCIP_CSHARP_HELPER_ENV, + helper.to_string_lossy().as_ref(), + )]) +} + +#[test] +fn current_git_head_resolves_the_crate_checkout() { + // The crate dir is always a git checkout (build.rs already depends on + // git metadata), so this is deterministic in dev and CI alike. + let head = crate::symbols::current_git_head(std::path::Path::new(env!("CARGO_MANIFEST_DIR"))); + let sha = head.expect("crate checkout must resolve a HEAD sha"); + assert_eq!(sha.len(), 40, "full sha expected: {sha}"); + assert!( + sha.chars().all(|c| c.is_ascii_hexdigit()), + "hex sha expected: {sha}" + ); +} + +#[test] +fn current_git_head_is_none_outside_a_repo() { + // A directory that is not a git repo must yield None, not an error — + // the fingerprint is best-effort by design. + let tmp = std::env::temp_dir().join("codesearch_no_git_head_check"); + let _ = std::fs::create_dir_all(&tmp); + assert!(crate::symbols::current_git_head(&tmp).is_none()); +} + +#[test] +fn dedupe_references_collapses_identical_entries_and_keeps_distinct_ones() { + // Live observation (2026-08-31): a definition arrived 5× — the SCIP + // find-refs output emits multiple occurrences at the same file:line for + // the declaring symbol. Identical (file, line range, kind) entries carry + // no separately-actionable information and collapse to the first. + let r = |file: &str, start: u32, end: u32, kind: &str| crate::symbols::SymbolReference { + file: std::path::PathBuf::from(file), + start_line: start, + end_line: end, + kind: kind.to_string(), + }; + let input = vec![ + r("src/A.cs", 9, 9, "definition"), + r("src/A.cs", 9, 9, "definition"), + r("src/A.cs", 9, 9, "definition"), + r("src/B.cs", 192, 192, "reference"), + r("src/A.cs", 9, 9, "definition"), + r("src/A.cs", 40, 44, "reference"), + r("src/A.cs", 9, 9, "reference"), // same line, DIFFERENT kind → kept + ]; + let out = super::dedupe_references(input); + let summary: Vec<(String, u32, u32, &str)> = out + .iter() + .map(|x| { + ( + x.file.to_string_lossy().into_owned(), + x.start_line, + x.end_line, + x.kind.as_str(), + ) + }) + .collect(); + assert_eq!( + summary, + vec![ + ("src/A.cs".to_string(), 9, 9, "definition"), + ("src/B.cs".to_string(), 192, 192, "reference"), + ("src/A.cs".to_string(), 40, 44, "reference"), + ("src/A.cs".to_string(), 9, 9, "reference"), + ], + "identical entries collapse to the first, distinct ones survive, order is stable: {summary:?}" + ); + + // Empty input stays empty (no panic on the with_capacity path). + assert!(super::dedupe_references(Vec::new()).is_empty()); +} + +#[tokio::test] +#[serial_test::serial] +async fn ambiguous_name_answers_with_the_typed_candidates_envelope() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + populate_overload_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("Validate".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["ambiguous"], + serde_json::Value::Bool(true), + "an overloaded name must answer the ambiguity envelope, got: {out}" + ); + let candidates = v["candidates"].as_array().expect("candidates array"); + assert_eq!(candidates.len(), 2, "both overloads listed: {v}"); + assert!( + candidates[0].as_str().unwrap() <= candidates[1].as_str().unwrap(), + "candidates must be sorted: {v}" + ); +} + +#[tokio::test] +#[serial_test::serial] +async fn an_explicit_key_that_misses_is_a_loud_failure_not_an_empty_answer() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + populate_overload_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some("csharp Ns . V#Gone().".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["class"], "failed", + "an explicit-key miss must be a typed failure, got: {out}" + ); + assert!( + v["error"].as_str().unwrap().contains("not in the"), + "the error must name the miss: {out}" + ); + assert!( + v.get("ambiguous").is_none(), + "a key miss is not ambiguity: {out}" + ); +} + +#[tokio::test] +#[serial_test::serial] +async fn fuzzy_not_found_keeps_the_empty_references_contract() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + populate_overload_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("Nope".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["references"], + serde_json::json!([]), + "the historical empty-references contract: {out}" + ); + assert_eq!(v["symbol"], "Nope", "the echo stays the query"); + assert!( + v.get("resolved_symbol").is_none(), + "an unresolved answer names no identity: {out}" + ); +} + +#[tokio::test] +async fn symbol_key_combined_with_a_fuzzy_query_is_rejected() { + // Validation precedes every lookup, so no helper env is needed. + let (service, _project) = build_service(); + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some("csharp Ns . V#Validate().".to_string()), + symbol_name: Some("Validate".to_string()), + ..find_impact_request() + }, + ) + .await; + assert!( + out.contains("mutually exclusive"), + "the combination must be rejected with the documented usage error: {out}" + ); +} + +/// The TS indexer passes `is_available` via the env override the same way +/// the C# one does — only `is_file()` is checked, so a dummy suffices. +fn make_ts_helper_available(root: &tempfile::TempDir) -> crate::testing::EnvRestore { + let helper = root.path().join(if cfg!(windows) { + "scip-typescript.exe" + } else { + "scip-typescript" + }); + std::fs::write(&helper, b"dummy").expect("dummy helper file"); + crate::testing::EnvRestore::set(&[( + crate::constants::SCIP_TYPESCRIPT_HELPER_ENV, + helper.to_string_lossy().as_ref(), + )]) +} + +#[tokio::test] +#[serial_test::serial] +async fn without_language_several_installed_helpers_ask_which_one() { + // Two helpers installed and no language: silently answering from the + // first would be the exact silent pick the ambiguity contract removes. + let csharp_root = tempfile::tempdir().unwrap(); + let ts_root = tempfile::tempdir().unwrap(); + let _csharp = make_helper_available(&csharp_root); + let _ts = make_ts_helper_available(&ts_root); + let (service, project) = build_service(); + populate_overload_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("Validate".to_string()), + language: None, + ..find_impact_request() + }, + ) + .await; + assert!( + out.contains("Several symbol indexes are installed"), + "the answer must ask which language, got: {out}" + ); + assert!( + out.contains("csharp") && out.contains("typescript"), + "the answer must list the installed languages: {out}" + ); + assert!( + !out.contains("\"ambiguous\""), + "asking for a language is not an ambiguity envelope: {out}" + ); +} + +#[tokio::test] +#[serial_test::serial] +async fn without_language_a_single_installed_helper_answers_deterministically() { + // Exactly one installed: the pick is deterministic, so the query + // proceeds (here: to the ambiguity envelope from the fixture) instead + // of asking which language to use. + // + // The premise only holds where `scip-typescript` is NOT resolvable: + // the adapter falls back to `npx` on PATH, so on a Node machine TS is + // always installed and the single-helper scenario does not exist. + let lookup = if cfg!(windows) { "where" } else { "which" }; + let npx_on_path = std::process::Command::new(lookup) + .arg("npx") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if npx_on_path { + eprintln!("skipped: npx on PATH makes TS installed, so several helpers are installed"); + return; + } + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + populate_overload_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("Validate".to_string()), + language: None, + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["ambiguous"], + serde_json::Value::Bool(true), + "one installed helper must answer, not ask: {out}" + ); +} + +// ── Warnings contract: a partial answer must say so ───────────────── +// +// B3 honesty: resolution warnings persisted in LMDB surface on the resolved +// answer; a clean answer must OMIT the field entirely (the additive-JSON +// contract pins `skip_serializing_if`). + +/// Version byte + bincode of `Vec<(PathBuf, u32, u32, String)>` — the same +/// bytes `serialize_refs` writes for a `StoredReference` (bincode encodes a +/// struct as its fields in declaration order, identical to the tuple). +fn stored_refs_bytes(file: &str, kind: &str) -> Vec { + let mut bytes = vec![1u8]; + bytes.extend_from_slice( + &bincode::serialize(&vec![( + std::path::PathBuf::from(file), + 7u32, + 9u32, + kind.to_string(), + )]) + .unwrap(), + ); + bytes +} + +/// Version byte + bincode of `Vec` — the same bytes the lazy and +/// batch write paths store in `scip_ref_warnings` (the key-list format). +fn warnings_bytes(warnings: &[&str]) -> Vec { + let owned: Vec = warnings.iter().map(|w| w.to_string()).collect(); + let mut bytes = vec![1u8]; + bytes.extend_from_slice(&bincode::serialize(&owned).unwrap()); + bytes +} + +/// One resolved symbol with persisted warnings, one without. +fn populate_warnings_fixture(db_path: &std::path::Path) -> (String, String) { + let warned_key = "csharp Ns . W#Warned().".to_string(); + let clean_key = "csharp Ns . C#Clean().".to_string(); + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + let symbols: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_SYMBOLS_DB_NAME)) + .unwrap() + .unwrap(); + for key in [&warned_key, &clean_key] { + symbols + .put( + &mut wtxn, + key.as_str(), + &stored_refs_bytes("src/w.cs", "definition"), + ) + .unwrap(); + } + let warnings_db: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_REF_WARNINGS_DB_NAME)) + .unwrap(); + warnings_db + .put( + &mut wtxn, + warned_key.as_str(), + &warnings_bytes(&[ + "FindReferencesAsync failed for Warned: InvalidOperationException: boom", + ]), + ) + .unwrap(); + wtxn.commit().unwrap(); + (warned_key, clean_key) +} + +#[tokio::test] +#[serial_test::serial] +async fn resolved_answer_surfaces_persisted_warnings() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + let (warned_key, _clean_key) = + populate_warnings_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some(warned_key.clone()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["resolved_symbol"], warned_key, + "the fixture query must actually resolve, got: {out}" + ); + let warnings = v["warnings"].as_array().expect("warnings array present"); + assert_eq!( + warnings.len(), + 1, + "the persisted warning must surface: {out}" + ); + assert!( + warnings[0] + .as_str() + .unwrap() + .contains("FindReferencesAsync failed for Warned"), + "warning text must round-trip verbatim: {out}" + ); +} + +#[tokio::test] +#[serial_test::serial] +async fn clean_resolved_answer_omits_the_warnings_field() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + let (_warned_key, clean_key) = + populate_warnings_fixture(&project.path().join(".codesearch.db")); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some(clean_key.clone()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["resolved_symbol"], clean_key, + "the fixture query must actually resolve, got: {out}" + ); + assert!( + v.get("warnings").is_none(), + "a clean answer must OMIT warnings (skip_serializing_if), got: {out}" + ); +} diff --git a/src/mcp/find_impact_tracker.rs b/src/mcp/find_impact_tracker.rs new file mode 100644 index 00000000..f742e737 --- /dev/null +++ b/src/mcp/find_impact_tracker.rs @@ -0,0 +1,184 @@ +//! Background continuation for budget-overrun `find_impact` lookups. +//! +//! When the `find_impact` wall-clock budget overruns, the answering handler +//! returns a structured busy envelope while the lookup keeps running as a +//! detached blocking task (its reference-cache writes still land in LMDB). +//! This module is the tracking half of that design: a registry keyed by +//! (project, symbol identity) so a retry observes current progress, or the +//! warm precise result once the detached lookup finished, instead of +//! racing a second identical helper subprocess against a cold cache. +//! +//! Semantics: +//! - `register` get-or-creates the entry: two racing first-callers share +//! one entry, so the second is answered from it (dedupe) rather than +//! starting a duplicate subprocess. +//! - A `Running` entry is reported with cumulative elapsed time. +//! - A `Finished` entry is consumed on read (removed from the map): exactly +//! one retry sees the warm result; later lookups go through the normal +//! path and hit the real reference cache, which is warm by then. +//! - Entries expire after a TTL (see `FIND_IMPACT_TRACK_TTL_SECS`), which +//! also covers a blocking task that died without recording (a panic in +//! the helper call): the next lookup after expiry starts fresh. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use crate::constants::FIND_IMPACT_TRACK_TTL_SECS; +use crate::symbols::SymbolReference; + +/// Identity of a tracked lookup: (project db dir, symbol identity string). +/// +/// The identity string is the same rendering the busy envelope shows — +/// `'Ns.I.M'` for name lookups, `file:line` for position lookups. Two +/// requests that spell the same symbol differently (e.g. absolute vs +/// relative file paths on the position variant) map to different keys; +/// that only weakens dedupe, never correctness. +pub(crate) type LookupKey = (PathBuf, String); + +/// Recorded outcome of a finished lookup. The error is the rendered +/// `{:#}` chain (`anyhow::Error` is not `Clone`); the typed failure +/// envelope is built from this at response time. +pub(crate) type RecordedResult = Result, String>; + +enum EntryState { + Running { + started: Instant, + }, + Finished { + result: RecordedResult, + finished: Instant, + }, +} + +/// One tracked lookup. Shared between the answering handler and the +/// detached blocking task via `Arc`; `finish` is called from INSIDE the +/// blocking task so the outcome is recorded even when the handler's +/// awaiting future was dropped at budget overrun. +pub(crate) struct LookupEntry { + state: Mutex, +} + +impl LookupEntry { + fn new() -> Self { + Self { + state: Mutex::new(EntryState::Running { + started: Instant::now(), + }), + } + } + + /// Record the lookup outcome. A panic inside the blocked call never + /// reaches this — such an entry stays `Running` until the TTL drops it. + pub(crate) fn finish(&self, result: RecordedResult) { + let mut state = self.lock(); + *state = EntryState::Finished { + result, + finished: Instant::now(), + }; + } + + fn lock(&self) -> std::sync::MutexGuard<'_, EntryState> { + // Poisoning can only come from a panic between lock and assignment, + // which is no state worth preserving: recover the inner guard. + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } +} + +/// What `ImpactLookupTracker::check` reports about a tracked lookup. +#[derive(Debug)] +pub(crate) enum TrackedStatus { + /// Still running; `elapsed_ms` is cumulative wall-clock since start. + Running { elapsed_ms: u64 }, + /// Finished — `Ok` is the warm precise result, `Err` the rendered + /// failure chain. Consumed on read. + Done(RecordedResult), +} + +/// Registry of in-flight / recently-finished `find_impact` lookups. +pub(crate) struct ImpactLookupTracker { + ttl: Duration, + entries: Mutex>>, +} + +impl ImpactLookupTracker { + pub(crate) fn new(ttl: Duration) -> Self { + Self { + ttl, + entries: Mutex::new(HashMap::new()), + } + } + + /// Observe a tracked lookup: progress while running, the outcome once + /// finished (consumed), or `None` when nothing is tracked (expired or + /// never started) so the caller starts a fresh lookup. + pub(crate) fn check(&self, key: &LookupKey) -> Option { + // Clone the entry out under a short map borrow; the state read and + // the consume-removal happen without holding the map lock. + let entry = { + let mut map = self.lock(); + self.sweep(&mut map); + map.get(key).cloned()? + }; + let state = entry.lock(); + match &*state { + EntryState::Running { started } => Some(TrackedStatus::Running { + elapsed_ms: elapsed_millis(started.elapsed()), + }), + EntryState::Finished { result, .. } => { + // Consume: the served retry is the one the busy advice + // addressed; everyone after it hits the warm reference + // cache through the normal path. + let result = result.clone(); + drop(state); + self.lock().remove(key); + Some(TrackedStatus::Done(result)) + } + } + } + + /// Get-or-create the entry for `key`. Racing first-callers share one + /// entry — both lookups record the same outcome into it, and whichever + /// finishes first answers (or the entry is consumed by a retry). + pub(crate) fn register(&self, key: LookupKey) -> Arc { + let mut map = self.lock(); + self.sweep(&mut map); + map.entry(key) + .or_insert_with(|| Arc::new(LookupEntry::new())) + .clone() + } + + /// Drop the entry after a lookup that finished WITHIN the budget: it + /// completed synchronously, nothing is in flight, and a later lookup + /// must consult the real cache rather than a remembered result. + pub(crate) fn remove(&self, key: &LookupKey) { + self.lock().remove(key); + } + + fn sweep(&self, map: &mut HashMap>) { + map.retain(|_, entry| match &*entry.lock() { + EntryState::Running { started } => started.elapsed() < self.ttl, + EntryState::Finished { finished, .. } => finished.elapsed() < self.ttl, + }); + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap>> { + self.entries.lock().unwrap_or_else(|p| p.into_inner()) + } +} + +fn elapsed_millis(d: Duration) -> u64 { + u64::try_from(d.as_millis()).unwrap_or(u64::MAX) +} + +/// Process-global tracker. Keys include the project db dir, so one global +/// serves every project without threading state through the service +/// constructors; the TTL bounds growth, and entries live only for the +/// duration of one lookup plus one retry. +pub(crate) static IMPACT_LOOKUP_TRACKER: LazyLock = + LazyLock::new(|| ImpactLookupTracker::new(Duration::from_secs(FIND_IMPACT_TRACK_TTL_SECS))); + +#[cfg(test)] +#[path = "find_impact_tracker_tests.rs"] +mod find_impact_tracker_tests; diff --git a/src/mcp/find_impact_tracker_tests.rs b/src/mcp/find_impact_tracker_tests.rs new file mode 100644 index 00000000..3cf1df29 --- /dev/null +++ b/src/mcp/find_impact_tracker_tests.rs @@ -0,0 +1,170 @@ +//! Tests for the `find_impact` background-continuation tracker. +//! +//! Behavioural, per-case tests against a locally constructed +//! `ImpactLookupTracker` (the process global is only a sharing wrapper — +//! the tracker logic carries no global state of its own). The one +//! cross-task test plants the exact production shape: a detached task that +//! finishes the entry after the registering caller stopped polling. + +use super::{ImpactLookupTracker, TrackedStatus}; +use crate::symbols::SymbolReference; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +fn key(name: &str) -> (PathBuf, String) { + (PathBuf::from("C:/proj/.codesearch.db"), name.to_string()) +} + +fn sample_refs() -> Vec { + vec![SymbolReference { + file: PathBuf::from("src/a.cs"), + start_line: 10, + end_line: 12, + kind: "call".to_string(), + }] +} + +#[test] +fn running_entry_reports_progress_then_warm_result_then_is_consumed() { + let tracker = ImpactLookupTracker::new(Duration::from_secs(60)); + let k = key("Ns.I.M"); + + let entry = tracker.register(k.clone()); + match tracker.check(&k) { + Some(TrackedStatus::Running { elapsed_ms }) => { + assert!(elapsed_ms <= 1_000, "just registered: {elapsed_ms}ms"); + } + other => panic!("a running entry must report Running, got {other:?}"), + } + + entry.finish(Ok(sample_refs())); + + match tracker.check(&k) { + Some(TrackedStatus::Done(Ok(refs))) => { + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].file, PathBuf::from("src/a.cs")); + assert_eq!(refs[0].kind, "call"); + } + other => panic!("a finished entry must serve the warm result, got {other:?}"), + } + // Consumed on read: the NEXT lookup must start fresh (real cache path). + assert!( + tracker.check(&k).is_none(), + "a Done entry must be consumed by its first reader" + ); +} + +#[test] +fn recorded_failure_is_served_once_with_rendered_chain() { + let tracker = ImpactLookupTracker::new(Duration::from_secs(60)); + let k = key("Gone.I.M"); + + let entry = tracker.register(k.clone()); + // The error side is recorded as its rendered `{:#}` chain (the render + // happens at the record site — `anyhow::Error` is not `Clone`). + let rendered = format!( + "{:#}", + anyhow::anyhow!("helper exited 1").context("scip-csharp find-refs failed") + ); + entry.finish(Err(rendered)); + + match tracker.check(&k) { + Some(TrackedStatus::Done(Err(chain))) => { + assert!( + chain.contains("scip-csharp find-refs failed") && chain.contains("helper exited 1"), + "the rendered chain must survive the tracker: {chain}" + ); + } + other => panic!("a recorded failure must surface as Done(Err), got {other:?}"), + } + assert!(tracker.check(&k).is_none(), "failure is consumed too"); +} + +#[test] +fn ttl_expiry_drops_entry_so_the_next_lookup_starts_fresh() { + // Windows timer granularity is ~15ms; 50ms TTL vs 250ms sleep keeps + // this deterministic on the slowest runner. + let tracker = ImpactLookupTracker::new(Duration::from_millis(50)); + let k = key("Stale.I.M"); + + tracker.register(k.clone()); + std::thread::sleep(Duration::from_millis(250)); + assert!( + tracker.check(&k).is_none(), + "an entry past its TTL must be swept, not reported Running forever" + ); +} + +#[tokio::test] +async fn detached_task_finish_is_observed_by_a_later_retry() { + // The production shape: the handler's awaiting future is dropped at + // budget overrun, but the detached blocking task keeps running and + // records the outcome via the shared entry. A retry issued afterwards + // must observe it. + let tracker = Arc::new(ImpactLookupTracker::new(Duration::from_secs(60))); + let k = key("Detached.I.M"); + let entry = tracker.register(k.clone()); + let entry_for_task = Arc::clone(&entry); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + entry_for_task.finish(Ok(sample_refs())); + }); + drop(entry); // the registering side lets go; the task survives + + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + loop { + match tracker.check(&k) { + Some(TrackedStatus::Done(Ok(refs))) => { + assert_eq!(refs.len(), 1); + break; + } + Some(TrackedStatus::Done(Err(chain))) => { + panic!("detached finish recorded a failure it must not have: {chain}"); + } + Some(TrackedStatus::Running { .. }) => {} + None => panic!("entry vanished before the detached finish landed"), + } + assert!( + tokio::time::Instant::now() < deadline, + "detached finish never observed within 3s" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + finisher.await.expect("finisher task must not panic"); +} + +#[test] +fn racing_first_callers_share_one_entry() { + let tracker = ImpactLookupTracker::new(Duration::from_secs(60)); + let k = key("Dup.I.M"); + + let a = tracker.register(k.clone()); + let b = tracker.register(k.clone()); + assert!( + Arc::ptr_eq(&a, &b), + "register must get-or-create: racing callers dedupe onto one entry" + ); + // One finish serves both callers; the first reader consumes it. + a.finish(Ok(sample_refs())); + assert!(matches!( + tracker.check(&k), + Some(TrackedStatus::Done(Ok(_))) + )); + assert!(tracker.check(&k).is_none()); +} + +#[test] +fn remove_after_in_budget_completion_forgets_the_entry() { + let tracker = ImpactLookupTracker::new(Duration::from_secs(60)); + let k = key("Fast.I.M"); + + let entry = tracker.register(k.clone()); + entry.finish(Ok(sample_refs())); + tracker.remove(&k); + assert!( + tracker.check(&k).is_none(), + "a within-budget completion must leave nothing tracked" + ); +} diff --git a/src/mcp/get_chunk.rs b/src/mcp/get_chunk.rs new file mode 100644 index 00000000..c32cdd59 --- /dev/null +++ b/src/mcp/get_chunk.rs @@ -0,0 +1,306 @@ +//! `get_chunk` tool. Extracted from `mod.rs` (todo #105) — the `#[tool]` +//! method registers through the per-module router merged in `mod.rs`'s +//! `merged_tool_router`. + +use super::*; +use rmcp::{ + handler::server::wrapper::Parameters, + model::{CallToolResult, ContentBlock}, + tool, tool_router, ErrorData as McpError, +}; + +#[tool_router(router = get_chunk_router, vis = "pub(crate)")] +impl CodesearchService { + #[tool( + description = "Retrieve the full content of a specific chunk by its ID, plus optional surrounding lines for context.\nUse this after search or explore to read the actual code without loading the whole file.\n\nUSE FOR: reading a specific function/class body after finding it via search.\nSet context_lines (default 0, max 20) to include lines before and after the chunk.\n\nIMPORTANT (multi-repo): chunk_ids are local to each repository and are NOT globally unique.\nWhen `project` is omitted in multi-repo mode, the tool scans all repositories for the chunk_id.\nIf found in exactly one repo, it is returned automatically. If found in multiple repos, an `ambiguous_chunk_id` error lists the candidates so you can retry with `project`." + )] + pub(crate) async fn get_chunk( + &self, + Parameters(request): Parameters, + ) -> Result { + tracing::info!( + "📥 get_chunk(chunk_id={}, project={:?})", + request.chunk_id, + request.project, + ); + + // Federation: a `chunk_ref` of the form "/:" + // (returned by a federated search result) fetches the chunk from a remote + // peer rather than the local index. The alias scopes the fetch to a single + // remote project so the multi-repo peer can disambiguate the chunk_id. + if let Some(chunk_ref) = request.chunk_ref.as_deref() { + return self + .federated_get_chunk(chunk_ref, request.context_lines) + .await; + } + + // Federated mounts: `project=/` routes to the peer's own + // project, exactly like search's project-level federation — chunk_ids + // are peer-local, so the fetch reuses the chunk_ref path with a + // synthetic "/:" ref. Local aliases ALWAYS win a name + // clash: only route remotely when the name is not a local project. + if let Some(proj) = request.project.as_deref() { + let cfg = self.federation_config(); + if cfg.resolve(proj).is_none() { + if let Some(crate::db_discovery::repos::Target::RemoteProject { + peer_name, + peer: _, + remote_alias, + }) = cfg.resolve_remote_project(proj) + { + let chunk_ref = format!("{peer_name}/{remote_alias}:{}", request.chunk_id); + return self + .federated_get_chunk(&chunk_ref, request.context_lines) + .await; + } + } + } + + // In multi-repo serve mode, require explicit project or group scope. + // Unscoped get_chunk would fan-out over all repos, opening all DBs unnecessarily. + // Consistent with search/find/explore which also require scope. + if request.project.is_none() && request.group.is_none() { + if let Some(ref serve_state) = self.serve_state { + let config = serve_state.config_snapshot(); + if config.repos.len() > 1 { + return Ok(CallToolResult::success(vec![ContentBlock::text( + self.format_scope_error(), + )])); + } + } + } + + // Resolve project/group routing — allow unscoped only for single-repo mode + let ctx = match self + .resolve_routing(&request.project, &request.group, true, "get_chunk") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + let mut clamped = false; + let mut context_lines = request.context_lines.unwrap_or(0); + if context_lines > 20 { + context_lines = 20; + clamped = true; + } + + // Stores that failed while looking up this chunk. get_chunk previously + // collapsed every `Err` into "not found", so during the read-only + // incident it would have reported every chunk in every vendor repo as + // missing — a confident, wrong answer. + let mut chunk_warnings: Vec = Vec::new(); + + // Look up chunk — multi-store: smart candidate detection for chunk_id collision. + // chunk_ids are local per database, not globally unique. When no project is specified + // and multiple stores are active, scan all stores to find which ones have this chunk_id. + let chunk = if let Some(ref sv) = ctx.stores_vec { + if sv.len() > 1 && request.project.is_none() { + // Smart candidate detection: find which stores actually contain this chunk_id + let mut candidates: Vec<(&Arc, String)> = Vec::new(); + let aliases = ctx.aliases(); + for (i, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.get_chunk(request.chunk_id) { + Ok(Some(_)) => { + // A store that HAS the chunk stays a candidate even if + // its alias is missing. `resolve_repo_stores_multi` + // keeps stores and aliases the same length, so this is + // unreachable today — but gating the push on + // `aliases.get(i)` meant a future break of that + // invariant would degrade to a silent auto-route rather + // than a loud one. The placeholder is per-index so two + // aliasless candidates stay distinguishable in + // `candidate_projects`. + let alias = aliases + .get(i) + .cloned() + .unwrap_or_else(|| format!("")); + candidates.push((store_arc, alias)); + } + Ok(None) => continue, + Err(ref e) => { + note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", e); + continue; + } + } + } + match candidates.len() { + 0 => { + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result( + format!( + "Chunk {} not found in any repository. Verify the \ + chunk_id and index state.", + request.chunk_id + ), + &chunk_warnings, + ), + )])); + } + 1 => { + // Exactly one store has this chunk_id — auto-route + let (store_arc, ref alias) = candidates[0]; + // Record tool call for the specific repo that served this chunk + if let Some(ref serve_state) = self.serve_state { + serve_state.record_tool_call(alias, "get_chunk"); + serve_state.touch_access(alias); + } + let store = store_arc.vector_store.read().await; + match store.get_chunk(request.chunk_id) { + Ok(c) => c, + Err(ref e) => { + push_store_warning( + &mut chunk_warnings, + &store_warning(alias, "chunk lookup", &format!("{e:#}")), + ); + None + } + } + } + _ => { + // Multiple stores have this chunk_id — ambiguous. + // + // `candidate_projects` reads as the complete list, so a + // store that failed to answer has to be declared: the + // right repo may be the one missing from it. + let candidate_names: Vec<&str> = + candidates.iter().map(|(_, a)| a.as_str()).collect(); + let payload = ambiguous_chunk_payload( + request.chunk_id, + &candidate_names, + &chunk_warnings, + ); + return Ok(CallToolResult::success(vec![ContentBlock::text( + payload.to_string(), + )])); + } + } + } else { + // Single store or project specified — direct lookup + let aliases = ctx.aliases(); + let mut found = None; + for (i, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.get_chunk(request.chunk_id) { + Ok(Some(c)) => { + found = Some(c); + break; + } + Ok(None) => continue, + // Do NOT abandon the remaining stores: one broken store + // says nothing about the others, and the chunk may well + // live in a healthy one. + Err(ref e) => { + note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", e); + continue; + } + } + } + found + } + } else { + match self + .with_vector_store_read_for( + |store| store.get_chunk(request.chunk_id), + ctx.stores.clone(), + ) + .await + { + Ok(c) => c, + Err(e) => { + push_store_warning( + &mut chunk_warnings, + &store_warning( + ctx.project_alias.as_deref().unwrap_or("unknown"), + "chunk lookup", + &format!("{e:#}"), + ), + ); + None + } + } + }; + + let mut chunk = match chunk { + Some(c) => c, + None => { + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result( + format!( + "Chunk {} not found. Verify the chunk_id and index state.", + request.chunk_id + ), + &chunk_warnings, + ), + )])); + } + }; + + // Prefix path with alias for multi-repo identification + chunk.path = ctx.prefix_result_path(&chunk.path); + + let mut context_before = None; + let mut context_after = None; + let mut note = None; + + if context_lines > 0 { + // Resolve relative chunk paths against project root (not process CWD). + let source_path = if Path::new(&chunk.path).is_absolute() { + PathBuf::from(&chunk.path) + } else { + self.project_path.join(&chunk.path) + }; + match tokio::fs::read_to_string(&source_path).await { + Ok(src) => { + let lines: Vec<&str> = src.lines().collect(); + if !lines.is_empty() { + let before_start = chunk.start_line.saturating_sub(context_lines); + let before_end = chunk.start_line.min(lines.len()); + if before_start < before_end { + context_before = Some(lines[before_start..before_end].join("\n")); + } + + let after_start = chunk.end_line.min(lines.len()); + let after_end = (chunk.end_line + context_lines).min(lines.len()); + if after_start < after_end { + context_after = Some(lines[after_start..after_end].join("\n")); + } + } + } + Err(_) => { + note = Some( + "source file not readable, returning indexed content only".to_string(), + ); + } + } + } + + let response = GetChunkResponse { + chunk_id: request.chunk_id, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + content: chunk.content, + context_before, + context_after, + context_lines_clamped: if clamped { Some(true) } else { None }, + note, + }; + + // The success path is the one that used to drop this, and it is the + // dangerous one: a confidently-returned chunk from a group where a store + // failed to answer looks exactly like a chunk from a healthy group. Same + // false negative as an empty result, harder to notice. + respond_with_object(&response, &chunk_warnings) + } +} diff --git a/src/mcp/graph.rs b/src/mcp/graph.rs new file mode 100644 index 00000000..c1e5b190 --- /dev/null +++ b/src/mcp/graph.rs @@ -0,0 +1,679 @@ +//! Import/dependent/similar internals — dispatch targets of the `find` and +//! `explore` tools, not tools of their own (no router). Extracted from +//! `mod.rs` (todo #105). + +use super::*; +use rmcp::model::{CallToolResult, ContentBlock}; + +impl CodesearchService { + pub(crate) fn normalize_symbol_query_path(&self, project_root: &Path, file: &Path) -> PathBuf { + if file.is_absolute() { + if let Ok(relative) = file.strip_prefix(project_root) { + return PathBuf::from(relative.to_string_lossy().replace('\\', "/")); + } + } + + PathBuf::from(file.to_string_lossy().replace('\\', "/")) + } + + pub(crate) async fn find_imports( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // In serve mode, use the resolved project root from alias_roots + let project_root = if let Some(ref alias) = ctx.project_alias { + ctx.alias_roots + .get(alias) + .map(PathBuf::from) + .unwrap_or_else(|| self.project_path.clone()) + } else { + self.project_path.clone() + }; + // Strip project-alias prefix from target path if present. + let stripped_path = strip_alias_prefix(&request.path, ctx.project_alias.as_ref()); + let normalized = normalize_tool_path(&stripped_path, &project_root); + + // Stores that failed during this lookup, so "no imports found" is never + // reported as fact when a store never answered. + let mut import_warnings: Vec = Vec::new(); + + let mut items = if let Some(ref sv) = ctx.stores_vec { + // Multi-store group fan-out: collect import items from all stores + let import_aliases = ctx.aliases(); + let mut all_items: Vec = Vec::new(); + let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.chunks_for_file(&normalized) { + Ok(metas) => { + for meta in metas { + if !is_import_kind(&meta.kind) { + continue; + } + if seen_ids.insert(meta.id) { + match store.get_chunk(meta.id) { + Ok(Some(chunk)) => all_items.extend(parse_import_lines( + &chunk.content, + chunk.start_line, + )), + Ok(None) => {} + Err(ref e) => note_store_failure( + &mut import_warnings, + import_aliases, + store_idx, + "chunk lookup", + e, + ), + } + } + } + } + Err(ref e) => { + note_store_failure( + &mut import_warnings, + import_aliases, + store_idx, + "imports scan", + e, + ); + } + } + } + all_items + } else { + match self + .with_vector_store_read_for( + |store| { + let mut out = Vec::new(); + for meta in store.chunks_for_file(&normalized)? { + if !is_import_kind(&meta.kind) { + continue; + } + if let Some(chunk) = store.get_chunk(meta.id)? { + out.extend(parse_import_lines(&chunk.content, chunk.start_line)); + } + } + Ok(out) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error reading imports: {e:#}" + ))])); + } + } + }; + + if items.is_empty() { + // Fallback: no import-kind chunks found for this file. Broaden the + // search to common import keywords and filter to the target path. + // Limitation: this only finds chunks containing these literal words; + // language-specific import forms that lack these keywords will be missed. + let fallback_limit = 40usize; + let mut all_hits: Vec<(u32, f32)> = Vec::new(); + let mut seen_fts_ids: HashSet = HashSet::new(); + + if let Some(ref sv) = ctx.stores_vec { + let import_aliases = ctx.aliases(); + // Multi-store FTS fallback + for keyword in IMPORT_FTS_KEYWORDS { + let hits = self + .with_fts_store_read_multi( + |fts_store| fts_store.search_exact(keyword, fallback_limit, None), + sv.clone(), + ctx.store_aliases.as_ref().unwrap(), + ) + .await + .unwrap_or_default() + .into_results(&mut import_warnings, "imports search"); + for h in hits { + if seen_fts_ids.insert(h.chunk_id) { + all_hits.push((h.chunk_id, h.score)); + } + } + } + + // Resolve FTS hits via vector stores + let mut resolved: Vec = Vec::new(); + for (chunk_id, _) in &all_hits { + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.get_chunk(*chunk_id) { + Ok(Some(chunk)) => { + if crate::cache::normalize_path_str(&chunk.path) == normalized { + resolved.extend(parse_import_lines( + &chunk.content, + chunk.start_line, + )); + } + break; + } + Ok(None) => continue, + Err(ref e) => { + note_store_failure( + &mut import_warnings, + import_aliases, + store_idx, + "chunk lookup", + e, + ); + continue; + } + } + } + } + items = resolved; + } else { + // Single-store FTS fallback + for keyword in IMPORT_FTS_KEYWORDS { + let hits = match self + .with_fts_store_read_for( + |fts_store| fts_store.search_exact(keyword, fallback_limit, None), + ctx.stores.clone(), + ) + .await + { + Ok(h) => h, + Err(e) => { + push_store_warning( + &mut import_warnings, + &store_warning( + ctx.project_alias.as_deref().unwrap_or("unknown"), + "imports search", + &format!("{e:#}"), + ), + ); + Vec::new() + } + }; + for h in hits { + if seen_fts_ids.insert(h.chunk_id) { + all_hits.push((h.chunk_id, h.score)); + } + } + } + + items = self + .with_vector_store_read_for( + |store| { + let mut out = Vec::new(); + for (chunk_id, _) in &all_hits { + if let Some(chunk) = store.get_chunk(*chunk_id)? { + if crate::cache::normalize_path_str(&chunk.path) == normalized { + out.extend(parse_import_lines( + &chunk.content, + chunk.start_line, + )); + } + } + } + Ok(out) + }, + ctx.stores.clone(), + ) + .await + .unwrap_or_else(|e| { + push_store_warning( + &mut import_warnings, + &store_warning( + ctx.project_alias.as_deref().unwrap_or("unknown"), + "chunk lookup", + &format!("{e:#}"), + ), + ); + Vec::new() + }); + } + } + + items.sort_by_key(|i| i.line); + respond_with_items(&items, &import_warnings, || { + "No import chunks found. The index may not include import statements \ + for this language, or the file has no imports." + .to_string() + }) + } + + pub(crate) async fn find_dependents( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "find") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + let limit = request.limit.unwrap_or(20).min(200); + let high_limit = (limit * 10).max(200); // generous budget for filtering + + // Stores that failed during this lookup, so "no dependents" is never + // reported as fact when a store never answered. + let mut dep_warnings: Vec = Vec::new(); + + // Extract a meaningful search term from path-like inputs. + // Import chunks contain module references like `use crate::constants::X` + // but the tool receives file paths like `src/constants.rs`. + // We extract the file stem to match against module names in imports. + let search_term = if request.symbol_or_path.contains('/') + || request.symbol_or_path.contains('\\') + || request.symbol_or_path.contains('.') + { + std::path::Path::new(&request.symbol_or_path) + .file_stem() + .and_then(|s| s.to_str()) + .filter(|s| !s.is_empty()) + .unwrap_or(&request.symbol_or_path) + .to_string() + } else { + request.symbol_or_path.clone() + }; + + let import_kind = Some(crate::chunker::ChunkKind::Imports); + + // Two-phase search strategy: + // 1. `search_exact` — precise term match on signature+content with + // MUST filter for Import kind. Strictly limits results to import chunks. + // 2. If that yields no import-kind results, fall back to `search` + // (QueryParser, broader tokenization) with kind boost for imports. + // + // Limitation: the chunker does not emit per-statement AST import chunks; + // imports are gap-classified as `Imports` kind. Chunks whose kind doesn't + // match `is_import_kind()` will be missed regardless of search method. + let fts_results = if let Some(ref sv) = ctx.stores_vec { + let sa = ctx.store_aliases.as_ref().unwrap(); + // Multi-store FTS search + let exact_hits = self + .with_fts_store_read_multi( + |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), + sv.clone(), + sa, + ) + .await + .unwrap_or_default() + .into_results(&mut dep_warnings, "dependents search"); + + if exact_hits.is_empty() { + self.with_fts_store_read_multi( + |fts_store| fts_store.search(&search_term, high_limit, import_kind), + sv.clone(), + sa, + ) + .await + .unwrap_or_default() + .into_results(&mut dep_warnings, "dependents search") + } else { + exact_hits + } + } else { + // Single-store FTS search + let alias = ctx.project_alias.as_deref().unwrap_or("unknown"); + let mut run = |r: anyhow::Result>| match r { + Ok(hits) => hits, + Err(e) => { + push_store_warning( + &mut dep_warnings, + &store_warning(alias, "dependents search", &format!("{e:#}")), + ); + Vec::new() + } + }; + let exact_hits = run(self + .with_fts_store_read_for( + |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), + ctx.stores.clone(), + ) + .await); + + if exact_hits.is_empty() { + run(self + .with_fts_store_read_for( + |fts_store| fts_store.search(&search_term, high_limit, import_kind), + ctx.stores.clone(), + ) + .await) + } else { + exact_hits + } + }; + + let mut items = if let Some(ref sv) = ctx.stores_vec { + // Multi-store: resolve chunks across all stores + let dep_aliases = ctx.aliases(); + let mut seen_paths = HashSet::new(); + let mut out = Vec::new(); + for f in &fts_results { + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.get_chunk(f.chunk_id) { + Ok(Some(chunk)) => { + if !is_import_kind(&chunk.kind) { + break; // try next FTS result + } + + let norm = crate::cache::normalize_path_str(&chunk.path); + if !seen_paths.insert(norm) { + break; + } + + let term_lower = search_term.to_lowercase(); + let import_statement = + if chunk.content.to_lowercase().contains(&term_lower) { + chunk + .content + .lines() + .find(|l| l.to_lowercase().contains(&term_lower)) + .unwrap_or("") + .to_string() + } else { + chunk.signature.filter(|s| !s.is_empty()).unwrap_or( + chunk.content.lines().next().unwrap_or("").to_string(), + ) + }; + + out.push(DependentItem { + path: chunk.path, + line: chunk.start_line, + import_statement, + }); + + break; // found in this store, move to next FTS result + } + Ok(None) => {} // try next store + // One broken store says nothing about the others; a + // `break` here silently drops a chunk that lives in a + // healthy store later in the list. + Err(ref e) => { + note_store_failure( + &mut dep_warnings, + dep_aliases, + store_idx, + "chunk lookup", + e, + ); + continue; + } + } + } + if out.len() >= limit { + break; + } + } + out + } else { + match self + .with_vector_store_read_for( + |store| { + let mut seen_paths = HashSet::new(); + let mut out = Vec::new(); + let term_lower = search_term.to_lowercase(); + for f in &fts_results { + if let Some(chunk) = store.get_chunk(f.chunk_id)? { + if !is_import_kind(&chunk.kind) { + continue; + } + + let norm = crate::cache::normalize_path_str(&chunk.path); + if !seen_paths.insert(norm) { + continue; + } + + // Extract the specific import line(s) that mention the + // module name, rather than returning the entire chunk content. + let import_statement = + if chunk.content.to_lowercase().contains(&term_lower) { + chunk + .content + .lines() + .find(|l| l.to_lowercase().contains(&term_lower)) + .unwrap_or("") + .to_string() + } else { + chunk.signature.filter(|s| !s.is_empty()).unwrap_or( + chunk.content.lines().next().unwrap_or("").to_string(), + ) + }; + + out.push(DependentItem { + path: chunk.path, + line: chunk.start_line, + import_statement, + }); + + if out.len() >= limit { + break; + } + } + } + Ok(out) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error resolving dependents: {e:#}" + ))])); + } + } + }; + + // Prefix paths with alias for multi-repo identification + for item in &mut items { + item.path = ctx.prefix_result_path(&item.path); + } + + items.sort_by(|a, b| a.path.cmp(&b.path)); + respond_with_items(&items, &dep_warnings, || { + format!("No dependent files found for '{}'.", request.symbol_or_path) + }) + } + + /// Internal: find similar chunks, used by `explore(kind="similar")`. + pub(crate) async fn similar_chunks( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "explore") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + let limit = request.limit.unwrap_or(5).min(20); + + // Stores that failed while resolving the source embedding. `if let + // Ok(Some(..))` used to discard the error, so a dead store produced + // "embedding not found" — a wrong diagnosis, not a missing chunk. + let mut similar_warnings: Vec = Vec::new(); + + let mut results = if let Some(ref sv) = ctx.stores_vec { + // Multi-store: find the embedding in whichever store has it, + // then search across all stores for similar chunks. + let aliases = ctx.aliases(); + let mut embedding: Option> = None; + for (i, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.get_embedding(request.chunk_id) { + Ok(Some(emb)) => { + embedding = Some(emb); + break; + } + Ok(None) => continue, + Err(ref e) => { + note_store_failure( + &mut similar_warnings, + aliases, + i, + "embedding lookup", + e, + ); + continue; + } + } + } + + let embedding = match embedding { + Some(e) => e, + None => { + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result( + format!( + "Embedding not found for chunk_id {} in any store.", + request.chunk_id + ), + &similar_warnings, + ), + )])); + } + }; + + // Search across all stores with the found embedding + let mut all_results: Vec = Vec::new(); + let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); + for (store_idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.search(&embedding, limit + 1) { + Ok(mut neighbors) => { + neighbors.retain(|r| r.id != request.chunk_id); + for r in neighbors { + if seen_ids.insert(r.id) { + all_results.push(SearchResultItem { + chunk_id: Some(r.id), + path: r.path, + start_line: r.start_line, + end_line: r.end_line, + kind: r.kind, + score: r.score, + signature: r.signature, + content: None, + context_prev: None, + context_next: None, + source: None, + chunk_ref: None, + }); + } + } + } + Err(ref e) => { + // The embedding was found, so the handler returns results + // either way; without this, a group query silently omits + // every neighbour from the broken repo. + note_store_failure( + &mut similar_warnings, + aliases, + store_idx, + "similarity search", + e, + ); + } + } + } + + all_results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + all_results.truncate(limit); + all_results + } else { + match self + .with_vector_store_read_for( + |store| { + let embedding = + store.get_embedding(request.chunk_id)?.ok_or_else(|| { + anyhow::anyhow!( + "embedding not found for chunk_id {}", + request.chunk_id + ) + })?; + + let mut neighbors = store.search(&embedding, limit + 1)?; + neighbors.retain(|r| r.id != request.chunk_id); + neighbors.truncate(limit); + + let items = neighbors + .into_iter() + .map(|r| SearchResultItem { + chunk_id: Some(r.id), + path: r.path, + start_line: r.start_line, + end_line: r.end_line, + kind: r.kind, + score: r.score, + signature: r.signature, + content: None, + context_prev: None, + context_next: None, + source: None, + chunk_ref: None, + }) + .collect::>(); + Ok(items) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error finding similar chunks: {e:#}" + ))])); + } + } + }; + + // Prefix paths with alias for multi-repo identification + for item in &mut results { + item.path = ctx.prefix_result_path(&item.path); + } + + // Every exit carries the channel: the earlier read sat in an + // early-return arm, so once an embedding was found, every failure + // recorded afterwards (the whole neighbour fan-out) was discarded. + respond_with_items(&results, &similar_warnings, || { + format!("No similar chunks found for chunk_id {}.", request.chunk_id) + }) + } +} diff --git a/src/mcp/helpers.rs b/src/mcp/helpers.rs new file mode 100644 index 00000000..49001c98 --- /dev/null +++ b/src/mcp/helpers.rs @@ -0,0 +1,145 @@ +use std::path::Path; + +// === Multi-store fan-out traits === + +/// Trait for types that have a chunk ID (used for deduplication in group fan-out). +pub(crate) trait HasChunkId { + fn chunk_id(&self) -> u32; +} + +/// Trait for types that have a relevance score (used for sorting in group fan-out). +pub(crate) trait HasScore { + fn score(&self) -> f32; +} + +impl HasChunkId for crate::vectordb::SearchResult { + fn chunk_id(&self) -> u32 { + self.id + } +} + +impl HasScore for crate::vectordb::SearchResult { + fn score(&self) -> f32 { + self.score + } +} + +impl HasChunkId for crate::fts::FtsResult { + fn chunk_id(&self) -> u32 { + self.chunk_id + } +} + +impl HasScore for crate::fts::FtsResult { + fn score(&self) -> f32 { + self.score + } +} + +// === Simple Glob Matcher === + +pub(crate) fn normalize_tool_path(path: &str, project_root: &Path) -> String { + let p = Path::new(path); + let resolved = if p.is_absolute() { + p.to_path_buf() + } else { + project_root.join(p) + }; + crate::cache::normalize_path_str(resolved.to_string_lossy().as_ref()) +} + +/// Strip a project-alias prefix from a tool path. +/// +/// In serve mode, tools like explore receive `target = "ALIAS/src/foo.rs"` with +/// `project = "ALIAS"`. The alias prefix must be stripped before calling +/// `chunks_for_file`, which expects a path relative to the project root. +pub(crate) fn strip_alias_prefix(path: &str, alias: Option<&String>) -> String { + if let Some(a) = alias { + let prefix = format!("{}/", a); + match path.strip_prefix(&prefix) { + Some(rest) => rest.to_string(), + None => path.to_string(), + } + } else { + path.to_string() + } +} + +/// Prefix a result path with its repo alias for group queries, normalizing +/// Windows backslashes to forward slashes in the process. When `alias` is +/// None or empty, the path is still normalized (useful for stdio mode). +pub(crate) fn prefix_path_with_alias( + path: &str, + alias: Option<&str>, + project_root: &str, +) -> String { + let normalized = crate::cache::normalize_path_str(path); + let normalized_root = crate::cache::normalize_path_str(project_root) + .trim_end_matches('/') + .to_string(); + match normalized.strip_prefix(&normalized_root) { + Some(rest) => { + let relative = rest.trim_start_matches('/'); + match alias { + Some(a) if !a.is_empty() => format!("{}/{}", a, relative), + _ => relative.to_string(), + } + } + None => normalized, + } +} + +/// Prefix a result path with the matching repo alias from a set of aliases and their roots. +/// Used by handlers that have alias/root info but not a full `MultiStoreContext`. +pub(crate) fn prefix_path_multi( + path: &str, + aliases: &[String], + alias_roots: &std::collections::HashMap, +) -> String { + let normalized = crate::cache::normalize_path_str(path); + for alias in aliases { + if let Some(root) = alias_roots.get(alias) { + if normalized.starts_with(root.as_str()) { + return prefix_path_with_alias(path, Some(alias), root); + } + } + } + normalized +} + +/// Pick the project root to relativise a result path against for a `filter_path` +/// prefix match, so `filter_path` is interpreted **relative to the repo root** +/// in every routing mode: +/// - serve single-project routing → the routed alias's root (`alias_roots[alias]`); +/// - serve multi/group → the longest alias root the (absolute) path lives under; +/// - stdio single-repo (no alias roots) → the service's own `project_path` +/// (`fallback_root`). +/// +/// Before this, the filter always used the service's `project_path`, which for a +/// serve-routed project is NOT the routed repo's root — so the absolute stored +/// path never relativised and every hit was dropped. The federated paths solve +/// the same class of bug client-side (see `retain_by_filter_path`); this covers +/// the local (non-federated) serve/multi case. +pub(crate) fn pick_filter_root( + path: &str, + project_alias: Option<&str>, + alias_roots: &std::collections::HashMap, + fallback_root: &str, +) -> String { + if let Some(alias) = project_alias { + if let Some(root) = alias_roots.get(alias) { + return root.clone(); + } + } + if !alias_roots.is_empty() { + let normalized = crate::cache::normalize_path_str(path); + if let Some(root) = alias_roots + .values() + .filter(|r| normalized.starts_with(r.as_str())) + .max_by_key(|r| r.len()) + { + return root.clone(); + } + } + fallback_root.to_string() +} diff --git a/src/mcp/instructions.rs b/src/mcp/instructions.rs new file mode 100644 index 00000000..42902910 --- /dev/null +++ b/src/mcp/instructions.rs @@ -0,0 +1,45 @@ +/// MCP server instructions template (pre-substitution). Kept as a named const so +/// the line-count and deprecated-alias tests can validate it directly without +/// fragile `include_str!` source-text searching or instantiating the service. +/// Substitution uses `str::replace` (not `format!`) because `format!` requires a +/// literal; the placeholders are unique tokens that don't appear in the prose. +/// See `test_instructions_max_50_lines` / `test_no_deprecated_tool_aliases`. +pub(crate) const INSTRUCTIONS_TEMPLATE: &str = r#"codesearch — semantic code search + symbol impact analysis. + +WHEN TO USE codesearch (prefer over grep/glob): + Good for: semantic or cross-file lookup, unknown file paths, symbol navigation, + "where is X implemented", "find usages of Y", "how does Z flow through the code" + Not for: a single known file (just read it), trivial one-line edits, + exact literal patterns where plain grep is faster + +SERVICE-MODE NOTES (codesearch serve, esp. on another host): + - Paths come from the SERVER's filesystem. Use get_chunk to read content; + don't try to open returned paths locally. + - Not every directory is indexed (e.g. .venv, node_modules, build/). If a + search returns nothing, the dir may be unindexed — ask, don't grep blindly. + +PICK THE RIGHT TOOL FOR THE TASK: + "who calls X?" / "what breaks if I rename X?" + → find_impact (precise SCIP call-graph; if no backend for the language, it says so → then use find kind="usages") + "find code about X" / "how does X work" / "show me X" + → search(mode="semantic") — concepts + synonyms + identifiers + exact syntax like Vec / foo = null / a::b + → search(mode="literal", regex=true) — patterns semantic can't match + "where is X defined?" / "what does file X import?" + → find(kind="definition" | "imports") + "show all symbols in file X" / "code like chunk Y" + → explore(kind="outline" | "similar") + read chunk content → get_chunk(chunk_id) + index health / repo list → status + +RULES: + - search(semantic) is the DEFAULT for code lookup. Don't skip it. + - For "who calls X" / impact analysis, try find_impact first; fall back to find(kind="usages") only if find_impact reports no backend. + - NEVER use literal as first search unless you need exact syntax. + - project or group is REQUIRED in multi-repo mode. + +Mode: {mode} +Project: {project} +Database: {db} ({exists}) +Model: {model} ({dims}d) +"#; diff --git a/src/mcp/literal_search.rs b/src/mcp/literal_search.rs new file mode 100644 index 00000000..b2edb2ce --- /dev/null +++ b/src/mcp/literal_search.rs @@ -0,0 +1,499 @@ +//! Literal (FTS-only, regex-aware) search internals — a dispatch target of +//! the `search` tool, not a tool of its own (no router). Extracted from +//! `mod.rs` (todo #105). + +use super::*; +use rmcp::model::{CallToolResult, ContentBlock}; + +impl CodesearchService { + pub(crate) async fn literal_search( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "search") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + let limit = request.limit.unwrap_or(20); + let output_format = request.format.as_deref().unwrap_or("json"); + + // Repos that failed during this search. Reported to the caller: an + // agent that never sees the server log cannot otherwise distinguish a + // broken store from a repo that holds no match. + let mut literal_warnings: Vec = Vec::new(); + + // Auto-regex promotion: detect code patterns that BM25 would destroy + let user_set_regex = request.regex.unwrap_or(false); + let user_set_phrase = request.phrase.unwrap_or(false); + let auto_promoted = + !user_set_regex && !user_set_phrase && looks_like_code_pattern(&request.query); + + let (effective_query, effective_regex) = if auto_promoted { + let escaped = regex::escape(&request.query); + // Relax whitespace to \s+ so "foo = null" → "foo\s+=\s+null" + // regex::escape does not escape spaces, so replace literal spaces. + let relaxed = escaped.replace(' ', r"\s+"); + (relaxed, true) + } else { + (request.query.clone(), user_set_regex) + }; + + tracing::debug!( + "MCP literal_search: query='{}', regex={:?}, phrase={:?}, limit={}, file_glob={:?}, language={:?}, format={}, multi={}", + request.query, request.regex, request.phrase, limit, + request.file_glob, request.language, output_format, ctx.is_multi + ); + + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // Pre-compute normalized project root for stripping absolute paths in glob matching + let lang_filter = request.language.clone(); + let glob_filter = request.file_glob.clone(); + let regex_enabled = effective_regex; + let snippet_regex = if regex_enabled { + Regex::new(&effective_query).ok() + } else { + None + }; + let project_root_normalized = { + let root = crate::cache::normalize_path_str(self.project_path.to_str().unwrap_or("")); + root.trim_end_matches('/').to_string() + }; + + // Decide: BM25 path (for anchorable queries) or scan path (for tokenless regex + // or disjunctive OR patterns like TODO|FIXME|HACK that BM25 treats as AND). + let tokenless_regex = regex_enabled + && snippet_regex.is_some() + && (!regex_has_anchorable_token(&effective_query) + || regex_has_disjunctive_or(&effective_query)); + + let mut items: Vec = if tokenless_regex { + // ── Scan path ────────────────────────────────────────────── + // Tokenless regex (e.g. \bfn\s+\w+) — BM25 cannot produce useful + // candidates. Scan all chunks sequentially, apply regex post-filter. + // Score is 0.0 for all results (no BM25 ranking applies). + tracing::debug!("literal_search: tokenless regex detected, using scan path"); + if let Some(ref sv) = ctx.stores_vec { + // Multi-store scan + let mut items: Vec = Vec::new(); + for store_arc in sv { + let store = store_arc.vector_store.read().await; + let all_chunks = match store.iter_all_chunks() { + Ok(chunks) => chunks, + Err(_) => continue, + }; + for (_, chunk) in all_chunks { + if let Some(ref lang) = lang_filter { + let file_lang = Language::from_path(std::path::Path::new(&chunk.path)); + if file_lang.name() != lang { + continue; + } + } + if let Some(ref glob) = glob_filter { + let relative_path = chunk + .path + .strip_prefix(&project_root_normalized) + .unwrap_or(&chunk.path) + .trim_start_matches('/'); + if !simple_glob_match(glob, relative_path) { + continue; + } + } + if let Some((match_offset, snippet)) = match_line_for_literal( + &chunk.content, + &effective_query, + snippet_regex.as_ref(), + ) { + let match_line = chunk.start_line + match_offset; + items.push(LiteralSearchResultItem { + path: chunk.path, + start_line: match_line, + end_line: match_line, + snippet, + score: 0.0, // No BM25 score — scan-path results are unranked + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, + signature: chunk.signature.filter(|s| !s.is_empty()), + }); + if items.len() >= limit { + break; + } + } + } + if items.len() >= limit { + break; + } + } + items + } else { + // Single-store scan + match self + .with_vector_store_read_for( + |store| { + let all_chunks = store.iter_all_chunks()?; + let mut items: Vec = Vec::new(); + for (_, chunk) in all_chunks { + if let Some(ref lang) = lang_filter { + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); + if file_lang.name() != lang { + continue; + } + } + if let Some(ref glob) = glob_filter { + let relative_path = chunk + .path + .strip_prefix(&project_root_normalized) + .unwrap_or(&chunk.path) + .trim_start_matches('/'); + if !simple_glob_match(glob, relative_path) { + continue; + } + } + if let Some((match_offset, snippet)) = match_line_for_literal( + &chunk.content, + &effective_query, + snippet_regex.as_ref(), + ) { + let match_line = chunk.start_line + match_offset; + items.push(LiteralSearchResultItem { + path: chunk.path, + start_line: match_line, + end_line: match_line, + snippet, + score: 0.0, // No BM25 score — scan-path results are unranked + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, + signature: chunk.signature.filter(|s| !s.is_empty()), + }); + if items.len() >= limit { + break; + } + } + } + Ok(items) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error scanning chunks: {e:#}" + ))])); + } + } + } + } else { + // ── BM25 path ────────────────────────────────────────────── + // Note: regex=true uses BM25 for candidates, then post-filters with the + // actual regex on raw content (Tantivy's RegexQuery only works on individual + // tokens, not raw text — underscores/punctuation cause empty results). + // + // When regex is enabled, strip metacharacters from the BM25 query so + // Tantivy gets clean tokens (e.g. "class Cache" instead of "class \w+Cache\b"). + let bm25_query = if regex_enabled { + let cleaned = extract_bm25_query_from_regex(&effective_query); + if cleaned.is_empty() { + effective_query.clone() + } else { + cleaned + } + } else { + effective_query.clone() + }; + let fts_results = if let Some(ref sv) = ctx.stores_vec { + let sa = ctx.store_aliases.as_ref().unwrap(); + let outcome = self + .with_fts_store_read_multi( + |fts_store| { + if request.phrase.unwrap_or(false) { + fts_store.search_phrase(&bm25_query, limit * 3) + } else { + fts_store.search(&bm25_query, limit * 3, None) + } + }, + sv.clone(), + sa, + ) + .await + .unwrap_or_default(); + for (alias, err) in &outcome.failures { + let msg = format!("repo '{alias}' literal search failed: {err}"); + tracing::error!("MCP: {}", msg); + literal_warnings.push(msg); + } + outcome.results + } else { + match self + .with_fts_store_read_for( + |fts_store| { + if request.phrase.unwrap_or(false) { + fts_store.search_phrase(&bm25_query, limit * 3) + } else { + fts_store.search(&bm25_query, limit * 3, None) + } + }, + ctx.stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching: {e:#}" + ))])); + } + } + }; + + // Resolve chunk metadata and apply post-filters + if let Some(ref sv) = ctx.stores_vec { + // Multi-store: resolve chunks from all stores + let mut items: Vec = Vec::new(); + 'outer: for fts_result in &fts_results { + let sa = ctx.store_aliases.as_ref().unwrap(); + for (idx, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + let looked_up = store.get_chunk(fts_result.chunk_id); + if let Err(ref e) = looked_up { + note_store_failure(&mut literal_warnings, sa, idx, "chunk lookup", e); + } + if let Some(chunk) = looked_up.ok().flatten() { + if let Some(ref lang) = lang_filter { + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); + if file_lang.name() != lang { + continue; + } + } + if let Some(ref glob) = glob_filter { + let relative_path = chunk + .path + .strip_prefix(&project_root_normalized) + .unwrap_or(&chunk.path) + .trim_start_matches('/'); + if !simple_glob_match(glob, relative_path) { + continue; + } + } + let match_info = match_line_for_literal( + &chunk.content, + &effective_query, + snippet_regex.as_ref(), + ); + if regex_enabled && match_info.is_none() { + continue; + } + let (match_offset, snippet) = match_info.unwrap_or_else(|| { + (0, chunk.content.lines().next().unwrap_or("").to_string()) + }); + let match_line = chunk.start_line + match_offset; + items.push(LiteralSearchResultItem { + path: chunk.path, + start_line: match_line, + end_line: match_line, + snippet, + score: fts_result.score, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, + signature: chunk.signature.filter(|s| !s.is_empty()), + }); + if items.len() >= limit { + break 'outer; + } + break; // Found in this store + } + } + } + items + } else { + match self + .with_vector_store_read_for( + |store| { + // Resolve chunk metadata first so a store `Err` + // propagates to the error arm below ("Error + // resolving search results") instead of silently + // dropping the hit — `Ok(None)` alone is a true + // miss ("chunk not in this store"). + let resolved: anyhow::Result> = fts_results + .iter() + .map(|fts_result| { + let chunk = store.get_chunk(fts_result.chunk_id)?; + Ok((chunk, fts_result.score)) + }) + .collect(); + let items: Vec = resolved? + .into_iter() + .filter_map(|(looked_up, score)| { + let chunk = looked_up?; + Some((chunk, score)) + }) + .filter(|(chunk, _)| { + if let Some(ref lang) = lang_filter { + let file_lang = + Language::from_path(std::path::Path::new(&chunk.path)); + if file_lang.name() != lang { + return false; + } + } + if let Some(ref glob) = glob_filter { + let relative_path = chunk + .path + .strip_prefix(&project_root_normalized) + .unwrap_or(&chunk.path) + .trim_start_matches('/'); + if !simple_glob_match(glob, relative_path) { + return false; + } + } + true + }) + .take(limit) + .filter_map(|(chunk, score)| { + let match_info = match_line_for_literal( + &chunk.content, + &effective_query, + snippet_regex.as_ref(), + ); + if regex_enabled && match_info.is_none() { + return None; + } + let (match_offset, snippet) = match_info.unwrap_or_else(|| { + (0, chunk.content.lines().next().unwrap_or("").to_string()) + }); + let match_line = chunk.start_line + match_offset; + Some(LiteralSearchResultItem { + path: chunk.path, + start_line: match_line, + end_line: match_line, + snippet, + score, + kind: if chunk.kind.is_empty() { + None + } else { + Some(chunk.kind) + }, + signature: chunk.signature.filter(|s| !s.is_empty()), + }) + }) + .collect(); + Ok(items) + }, + ctx.stores.clone(), + ) + .await + { + Ok(items) => items, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error resolving search results: {e:#}" + ))])); + } + } + } + }; + + // Prefix paths with alias for multi-repo identification + for item in &mut items { + item.path = ctx.prefix_result_path(&item.path); + } + + // Compute low-confidence signal + let top_score = items.first().map(|i| i.score); + let (low_confidence, suggested_tool) = + compute_literal_low_confidence(top_score, &request.query); + + // Build note + let note = if auto_promoted { + Some(format!( + "Query auto-promoted to regex mode (original: '{}', effective: '{}'). \ + The query contained code-like punctuation that BM25 would tokenize incorrectly.", + request.query, effective_query + )) + } else if low_confidence == Some(true) { + suggested_tool.as_ref().map(|tool| { + format!( + "Top result has weak BM25 score; consider using `{}` for better matches.", + tool + ) + }) + } else { + None + }; + + let response = LiteralSearchResponse { + results: items, + auto_promoted_to_regex: if auto_promoted { Some(true) } else { None }, + note, + low_confidence, + suggested_tool: if low_confidence == Some(true) { + suggested_tool + } else { + None + }, + warnings: if literal_warnings.is_empty() { + None + } else { + Some(literal_warnings) + }, + }; + + // Instrument BM25 score for threshold calibration + if let Some(top) = response.results.first() { + tracing::debug!( + target: "codesearch::literal_confidence", + query = %request.query, + top_bm25_score = top.score, + result_count = response.results.len(), + "literal_search score sample" + ); + } + + // Format output + let output = if output_format == "grep" { + let mut lines: Vec = Vec::new(); + if response.auto_promoted_to_regex == Some(true) { + lines.push( + "# auto-promoted to regex mode (query contained code-like punctuation)" + .to_string(), + ); + } + if response.low_confidence == Some(true) { + if let Some(ref hint) = response.suggested_tool { + lines.push(format!("# low confidence — consider: {}", hint)); + } + } + for item in &response.results { + lines.push(format!( + "{}:{}:{}", + item.path, item.start_line, item.snippet + )); + } + lines.join("\n") + } else { + serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()) + }; + + Ok(CallToolResult::success(vec![ContentBlock::text(output)])) + } +} diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 72f25ed2..66d79ef0 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -28,10 +28,10 @@ fn serve_url_from_env() -> String { } use crate::db_discovery::{find_best_database, load_repos_config}; -use crate::embed::{EmbeddingService, ModelType}; +use crate::embed::{EmbeddingServicePool, ModelType}; use crate::file::Language; use crate::fts::FtsStore; -use crate::index::{IndexManager, SharedStores}; +use crate::index::SharedStores; use crate::rerank::{rrf_fusion, rrf_fusion_with_exact, vector_only, EXACT_MATCH_RRF_K}; use crate::search::{adapt_rrf_k, boost_kind, detect_identifiers, detect_structural_intent}; use crate::symbols::SymbolIndexerRegistry; @@ -41,645 +41,105 @@ use regex::Regex; use rmcp::{ handler::server::router::tool::ToolRouter, handler::server::wrapper::Parameters, - model::{ - CallToolRequestParams, CallToolResult, Content, Implementation, ListToolsResult, - PaginatedRequestParams, ServerCapabilities, ServerInfo, - }, - service::RequestContext, - tool, tool_handler, tool_router, ErrorData as McpError, RoleClient, RoleServer, ServerHandler, + model::{CallToolResult, ContentBlock, Implementation, ServerCapabilities, ServerInfo}, + tool_handler, tool_router, ErrorData as McpError, ServerHandler, }; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; -use tokio_util::sync::CancellationToken; // Re-export types pub use types::*; -// ═══════════════════════════════════════════════════════════════════ -// MCP Proxy Service (--mode client / --mode auto with serve detected) -// ═══════════════════════════════════════════════════════════════════ - -/// Transparent stdio↔HTTP proxy with automatic reconnect. -/// -/// When `codesearch mcp --mode client` is started by Claude Desktop: -/// - Claude Desktop sends MCP requests over stdio -/// - `McpProxyService` forwards every request to the running `codesearch serve` hub via HTTP -/// - Responses flow back unchanged -/// -/// This is the correct architecture for Claude Desktop: it has no repo context of its own -/// and therefore cannot use `--mode local`. With `--mode client` it always connects to -/// the serve hub, gaining access to all registered repos. -/// -/// Only tool operations (`list_tools`, `call_tool`) are forwarded. Prompts, resources, -/// and completion are not proxied — the serve hub does not expose them. -/// -/// ## Reconnect -/// -/// The peer is wrapped in `Arc>>` so it can be hot-swapped when the -/// serve connection drops and reconnects. During reconnection, tool calls return a -/// descriptive "reconnecting" error so Claude Desktop can retry. -/// -/// ## Idle disconnect / connect on demand -/// -/// The peer is also `None` while the proxy is *deliberately* disconnected: after -/// `CODESEARCH_MCP_PROXY_IDLE_DISCONNECT_SECS` (default -/// `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`, `0` disables) without a successful -/// forwarded request, the idle-checker in `run_mcp_client` closes the HTTP MCP -/// session so a scale-to-zero remote can suspend its replica. Every successful -/// `list_tools` / `call_tool` stamps `last_activity`, which resets that window. -/// -/// Because a closed session is indistinguishable from a dead one at the peer -/// slot, `call_tool` / `list_tools` signal `connect_request_tx` on their first -/// attempt whenever the slot is `None`, asking the main loop to connect *now* -/// instead of waiting for the failure-path reconnect cadence. The existing -/// bounded retry-with-backoff remains the fallback if that connect does not land -/// within the retry budget. -struct McpProxyService { - /// Shared peer handle — hot-swapped on reconnect. - /// `None` means we're reconnecting to serve; tool calls return a retry-able error. - peer: std::sync::Arc>>>, - /// Signal to the main loop in `run_mcp_client` that the current peer is dead - /// and a fresh `connect_to_serve` should be attempted. Sent from `call_tool` / - /// `list_tools` when rmcp returns a transport-level error so we can recover - /// from server restarts and TCP keep-alive failures without bubbling the error - /// up to Claude Desktop. - disconnect_tx: tokio::sync::mpsc::Sender<()>, - /// Ask the main loop to run `connect_to_serve` immediately (capacity-1 - /// channel — duplicate requests coalesce, "connect now" is idempotent). - /// Sent when a request arrives while the peer slot is empty. - connect_request_tx: tokio::sync::mpsc::Sender<()>, - /// When the last request was successfully forwarded to serve. Shared with the - /// idle-checker in `run_mcp_client`, which closes the connection once this is - /// older than the configured idle-disconnect window. - last_activity: Arc>, - /// Number of requests currently being forwarded. `last_activity` only advances - /// on completion, so without this a request that runs longer than the idle - /// window (a big search, a cold symbol rebuild) would have its own transport - /// closed underneath it. The idle-checker never disconnects while this is > 0. - in_flight: Arc, - /// Notified by the main loop's `connect_request_rx` arm whenever an on-demand - /// `connect_to_serve` attempt returns `Err` — i.e. serve refused the - /// connection outright, as opposed to still being slow to accept one. Lets - /// `await_peer` stop waiting immediately on a definitive failure instead of - /// polling out the rest of `PROXY_CONNECT_WAIT_MS` (previously ~20s per call - /// even when serve was known to be down within the first few milliseconds). - /// A slow-but-eventually-successful wake never touches this: it resolves by - /// the peer slot filling in, which `await_peer`'s own poll already catches. - connect_failed: Arc, -} - -/// Keeps `McpProxyService::in_flight` incremented for its lifetime. A guard rather -/// than paired add/sub calls because the forwarding loop has several early returns. -struct InFlightGuard(Arc); - -impl InFlightGuard { - fn new(counter: &Arc) -> Self { - counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - Self(counter.clone()) - } -} - -impl Drop for InFlightGuard { - fn drop(&mut self) { - self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - } -} - -impl McpProxyService { - #[allow(dead_code)] - fn new(peer: rmcp::service::Peer) -> Self { - // Direct constructor used by tests / single-shot scenarios. - // No reconnect plumbing — the dummy channels are never read. - let (tx, _rx) = tokio::sync::mpsc::channel(1); - let (connect_tx, _connect_rx) = tokio::sync::mpsc::channel(1); - Self { - peer: std::sync::Arc::new(tokio::sync::RwLock::new(Some(peer))), - disconnect_tx: tx, - connect_request_tx: connect_tx, - last_activity: Arc::new(Mutex::new(std::time::Instant::now())), - in_flight: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - connect_failed: Arc::new(tokio::sync::Notify::new()), - } - } - - /// Stamp "real traffic just flowed", resetting the idle-disconnect window. - fn mark_activity(&self) { - mark_proxy_activity(&self.last_activity); - } - - /// Best-effort nudge to the main loop: connect to serve now. A full channel - /// already means "a connect is pending", a closed one means the loop is gone; - /// both are fine to ignore — the caller's own retry/backoff covers it. - fn request_connect(&self) { - let _ = self.connect_request_tx.try_send(()); - } - - /// Wait — bounded by `PROXY_CONNECT_WAIT_MS` — for the peer slot to be filled - /// after `request_connect`. Returns true as soon as a peer is available. - /// - /// Without this, a request arriving after an idle-close would burn its whole - /// retry budget (~1s) while the on-demand connect is still waking a - /// scaled-to-zero remote, and fail with "reconnecting" every single time. - async fn await_peer(&self) -> bool { - self.await_peer_bounded(PROXY_CONNECT_WAIT_MS).await - } - - /// Core of `await_peer`, parameterized on the wait budget so it is unit - /// testable without actually waiting out `PROXY_CONNECT_WAIT_MS` (~20s). - /// Uses the production refusal-grace window; see - /// `await_peer_bounded_with_grace` for what that means and why it is its - /// own parameter. - async fn await_peer_bounded(&self, wait_ms: u64) -> bool { - self.await_peer_bounded_with_grace(wait_ms, CONNECT_REFUSAL_GRACE) - .await - } - - /// Core of `await_peer_bounded`, additionally parameterized on the - /// refusal-grace window so *that* is unit testable without waiting out - /// `reconnect::INTERVAL_SECS` (~3s) for real. - /// - /// Polls the peer slot on `PROXY_RETRY_BACKOFF_MS` cadence, but also races - /// each poll against `connect_failed` so a definitive on-demand connect - /// failure (serve refused the connection, not merely slow to accept one) - /// clamps the remaining wait down to `refusal_grace` instead of polling - /// out the rest of `wait_ms`. A slow-but-still-in-progress wake never - /// fires `connect_failed` — it is only notified from an `Err` return of - /// `connect_to_serve` — so this does not shorten the legitimate - /// scale-to-zero wake path, only the case where serve is already known to - /// have refused this attempt. - /// - /// The clamp is deliberately *not* an immediate return: a refusal only - /// means this one on-demand attempt was refused, not that serve won't - /// recover — `run_mcp_client`'s own disconnect/reconnect cycle - /// (`reconnect::INTERVAL_SECS` later) can still land within the original - /// budget, e.g. when serve is mid-restart rather than genuinely down. - /// Returning immediately turned that case — previously transparent to the - /// caller, since the pre-fix full-budget poll caught the reconnect — into - /// a visible "reconnecting" error on the very first request after a - /// restart. Clamping to `refusal_grace` keeps most of the original fix's - /// win (a hard-down serve is still bounded well under the full `wait_ms`) - /// while still giving that recovery cycle room to land. - async fn await_peer_bounded_with_grace( - &self, - wait_ms: u64, - refusal_grace: std::time::Duration, - ) -> bool { - let mut deadline = std::time::Instant::now() + std::time::Duration::from_millis(wait_ms); - loop { - // Register for the next failure notification BEFORE checking the - // peer slot, so a failure landing between this check and the - // `select!` below cannot be missed (the standard tokio::sync::Notify - // idiom: create the `Notified` future first, await it second). - let failed = self.connect_failed.notified(); - if self.peer.read().await.is_some() { - return true; - } - let now = std::time::Instant::now(); - if now >= deadline { - return false; - } - let backoff = std::time::Duration::from_millis(PROXY_RETRY_BACKOFF_MS) - .min(deadline.saturating_duration_since(now)); - tokio::select! { - _ = tokio::time::sleep(backoff) => {} - _ = failed => { - // A concurrent successful connect could still have landed in - // the instant before this notification; one last check keeps - // that case correct instead of reporting a false failure. - if self.peer.read().await.is_some() { - return true; - } - deadline = deadline.min(now + refusal_grace); - } - } - } - } - - /// On an empty peer slot (a deliberate idle-close or a real outage), ask the - /// main loop to connect *now* instead of waiting for the failure-path - /// reconnect cadence to notice, then give that connect a bounded window to - /// land. Returns true if a peer became available and the caller should - /// retry its forwarded call immediately. - /// - /// Only meaningful on the caller's first attempt (`attempt == 0`) — a - /// second empty slot means the on-demand connect already ran and fell - /// through to the ordinary retry/backoff path. Pulled out of `list_tools`/ - /// `call_tool` because the two copies had already started to drift (see - /// review remarks on the commit that added this). - async fn try_on_demand_connect(&self) -> bool { - self.request_connect(); - self.await_peer().await - } - - /// Force a reconnect: clear the shared peer and signal the main loop in - /// `run_mcp_client` to call `connect_to_serve` again. Brief sleep gives - /// the main loop time to actually reconnect before the caller retries. - async fn force_reconnect(&self) { - *self.peer.write().await = None; - let _ = self.disconnect_tx.send(()).await; - tokio::time::sleep(std::time::Duration::from_millis( - crate::mcp::PROXY_RETRY_BACKOFF_MS, - )) - .await; - } +mod explore; +mod federation_helpers; +mod find; +mod find_impact; +mod get_chunk; +mod graph; +mod helpers; +mod instructions; +mod literal_search; +mod proxy; +mod responses; +mod runtime; +mod search; +mod status; + +// Re-export the extracted modules' items so every existing path keeps working: +// `super::X` from sibling test files and `crate::mcp::X` from serve/cli. +pub(crate) use federation_helpers::*; +pub(crate) use find_impact::rest_find_impact_handler; +pub(crate) use helpers::*; +pub(crate) use instructions::*; +pub(crate) use responses::*; +pub use runtime::*; + +/// Re-order lexical `usages` hits so source-code paths come before everything +/// else (docs, configs, markdown). Stable: score order is preserved within +/// each group, so this only demotes non-code noise, it never re-ranks code. +fn rank_code_first(items: &mut [ReferenceItem]) { + items.sort_by_key(|item| !is_source_path(&item.path)); +} + +/// True when the path looks like source code rather than docs/config. Used +/// only to re-order lexical `find(kind="usages")` hits code-first — never to +/// filter them: a markdown hit is noise the agent can discard, a missing hit +/// would be a silent false negative. +fn is_source_path(path: &str) -> bool { + const SOURCE_EXTS: &[&str] = &[ + "rs", "py", "go", "cs", "ts", "tsx", "js", "jsx", "mts", "cts", "mjs", "cjs", "java", "kt", + "kts", "swift", "c", "h", "cpp", "hpp", "cc", "cxx", "hh", "rb", "php", "proto", "scala", + "dart", + ]; + Path::new(path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| SOURCE_EXTS.contains(&e.to_lowercase().as_str())) + .unwrap_or(false) } -/// Maximum number of attempts when forwarding a request to serve. -/// Each retry includes a forced reconnect, so this also bounds reconnect attempts -/// per individual tool call. -const PROXY_MAX_RETRY_ATTEMPTS: u32 = 3; - -/// Backoff between proxy retries, also used as the post-reconnect settle delay. -const PROXY_RETRY_BACKOFF_MS: u64 = 500; - -/// How long a request may wait for an on-demand connect (after an idle-close, or -/// while serve is still starting) before falling back to the retry/backoff path. -/// -/// Sized for a scale-to-zero host: the remote's ingress *holds* the request while -/// it activates a suspended replica, so the connect itself can legitimately take -/// several seconds. Waiting here is strictly better than returning "reconnecting" -/// on the first call after every idle period. -const PROXY_CONNECT_WAIT_MS: u64 = 20_000; - -/// How long `await_peer_bounded` still waits after a definitive on-demand -/// connect refusal, instead of returning immediately or polling out the rest -/// of `PROXY_CONNECT_WAIT_MS`. -/// -/// Sized to cover `run_mcp_client`'s own disconnect/reconnect cycle -/// (`reconnect::INTERVAL_SECS`, ~3s) plus margin for the ~100ms synthetic- -/// disconnect delay and `connect_to_serve`'s own latency — so a serve that is -/// merely mid-restart still recovers transparently within this window, -/// exactly as it did before the refusal short-circuit existed, while a -/// genuinely-down serve is still bounded well under the full ~20s budget. -const CONNECT_REFUSAL_GRACE: std::time::Duration = - std::time::Duration::from_millis(reconnect::INTERVAL_SECS * 1_000 + 1_000); - -/// Record a definitive on-demand connect refusal: wake any `await_peer_bounded` -/// callers immediately (via `connect_failed`) instead of leaving them to poll -/// out their full budget for a refusal that is already known, then seed a -/// synthetic disconnect so `run_mcp_client`'s own disconnect/reconnect cycle -/// picks it up. A genuinely slow wake never reaches this function — it -/// resolves via the `Ok` branch in the caller once the peer slot fills in — -/// so this does not shorten a legitimate scale-to-zero wake, only a refusal. -/// -/// Pulled out of `run_mcp_client`'s `connect_request_rx` arm so the one line -/// that makes `await_peer_bounded`'s refusal short-circuit real in production -/// is covered by a test that calls this function directly, not only by tests -/// that call `connect_failed.notify_waiters()` themselves in isolation — -/// those pin how `await_peer_bounded` *reacts* to a notification, but nothing -/// previously pinned that this call site still *fires* one: deleting this -/// function's body left the full suite green. -fn note_connect_failure( - connect_failed: &tokio::sync::Notify, - disconnect_tx: &tokio::sync::mpsc::Sender<()>, -) { - connect_failed.notify_waiters(); - let tx = disconnect_tx.clone(); - tokio::spawn(async move { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let _ = tx.send(()).await; +/// Agent-facing note for lexical `find(kind="usages")` results. +/// +/// Returns `Some(note)` only when BOTH hold: the hits actually include +/// SCIP-backed source files (C#/TypeScript) AND a matching symbol indexer is +/// installed and available — precisely the case where the lexical list is a +/// lossy stand-in for `find_impact`'s precise references. Any other +/// combination returns `None`, keeping the legacy response shape +/// byte-identical (no nagging where the advice cannot be acted on). +fn scip_usages_note( + registry: &SymbolIndexerRegistry, + items: &[ReferenceItem], + symbol: &str, +) -> Option { + const SCIP_BACKED_EXTS: &[&str] = &["cs", "ts", "tsx", "mts", "cts"]; + let has_backed_source = items.iter().any(|item| { + Path::new(&item.path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| SCIP_BACKED_EXTS.contains(&e.to_lowercase().as_str())) + .unwrap_or(false) }); -} - -/// Heuristic: does this error message describe a transport-level failure -/// (broken TCP, server gone, stale keep-alive, stale session) that warrants -/// a forced reconnect + retry, as opposed to a real tool-level error that -/// the caller should see? -fn is_transport_error_msg(msg: &str) -> bool { - msg.contains("Transport send error") - || msg.contains("error sending request") - || msg.contains("Transport error") - || msg.contains("connection closed") - || msg.contains("error decoding response body") - || msg.contains("Session not found") - || msg.contains("404") -} - -/// Reconnect-related constants for the MCP proxy. -mod reconnect { - /// How long to wait between reconnect attempts. - pub const INTERVAL_SECS: u64 = 3; - /// Maximum total time to spend trying to reconnect before giving up. - pub const MAX_DURATION_SECS: u64 = 300; // 5 minutes -} - -/// Record the current instant as the proxy's most recent activity. -fn mark_proxy_activity(last_activity: &Arc>) { - if let Ok(mut slot) = last_activity.lock() { - *slot = std::time::Instant::now(); - } -} - -/// Resolve the MCP proxy idle-disconnect window: explicit value → env var → -/// `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`. Mirrors how `run_serve` resolves its -/// own `idle_suspend_secs`. `0` means "never idle-disconnect". -fn resolve_proxy_idle_disconnect_secs(explicit: Option) -> u64 { - explicit - .or_else(|| { - std::env::var(crate::constants::MCP_PROXY_IDLE_DISCONNECT_SECS_ENV) - .ok() - .and_then(|s| s.trim().parse().ok()) - }) - .unwrap_or(crate::constants::DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS) -} - -/// Has the proxy been idle long enough to close its connection to serve? -/// -/// `threshold_secs == 0` disables idle-disconnect, so this always returns false. -/// `now` is a parameter (rather than read from the clock) purely so this is unit -/// testable without sleeping. -fn is_idle( - last_activity: std::time::Instant, - threshold_secs: u64, - now: std::time::Instant, -) -> bool { - if threshold_secs == 0 { - return false; - } - now.saturating_duration_since(last_activity).as_secs() >= threshold_secs -} - -#[cfg(test)] -#[path = "proxy_idle_tests.rs"] -mod proxy_idle_tests; - -/// Unit tests for `await_peer_bounded`'s refusal clamp and the -/// `note_connect_failure` call site that fires it in production, isolated -/// from the full `run_mcp_client` loop by parameterizing the wait budget (and, -/// for the clamp itself, the refusal-grace window) so these run in -/// milliseconds instead of the real `PROXY_CONNECT_WAIT_MS` (~20s) or -/// `reconnect::INTERVAL_SECS` (~3s). -#[cfg(test)] -#[path = "await_peer_tests.rs"] -mod await_peer_tests; - -impl ServerHandler for McpProxyService { - fn get_info(&self) -> ServerInfo { - ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) - .with_server_info( - Implementation::new("codesearch", env!("CARGO_PKG_VERSION")) - .with_title("codesearch (serve proxy)"), - ) - .with_instructions( - "Proxy to a running codesearch serve hub. All tool calls are forwarded to the hub.", - ) - } - - async fn list_tools( - &self, - request: Option, - _cx: RequestContext, - ) -> Result { - let _in_flight = InFlightGuard::new(&self.in_flight); - let mut last_err: Option = None; - for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { - let peer = self.peer.read().await.clone(); - match peer { - Some(p) => match p.list_tools(request.clone()).await { - Ok(r) => { - self.mark_activity(); - return Ok(r); - } - Err(e) => { - let msg = e.to_string(); - if !is_transport_error_msg(&msg) || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 - { - return Err(McpError::internal_error(msg, None)); - } - tracing::warn!( - "list_tools attempt {}/{} failed (transport): {} — forcing reconnect", - attempt + 1, - PROXY_MAX_RETRY_ATTEMPTS, - msg - ); - last_err = Some(msg); - self.force_reconnect().await; - } - }, - None => { - // Empty peer slot: either a deliberate idle-close or a real - // outage. `try_on_demand_connect` asks the main loop to - // connect *now* rather than waiting for the failure-path - // reconnect cadence to notice, bounded so we still fall back - // to the ordinary retry/backoff below if it doesn't land. - if attempt == 0 && self.try_on_demand_connect().await { - continue; - } - if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { - tokio::time::sleep(std::time::Duration::from_millis( - PROXY_RETRY_BACKOFF_MS, - )) - .await; - continue; - } - return Err(McpError::internal_error( - "codesearch serve is reconnecting — please retry in a moment".to_string(), - None, - )); - } - } - } - Err(McpError::internal_error( - last_err.unwrap_or_else(|| "transport error after retries".to_string()), - None, - )) - } - - async fn call_tool( - &self, - request: CallToolRequestParams, - _cx: RequestContext, - ) -> Result { - let _in_flight = InFlightGuard::new(&self.in_flight); - let mut last_err: Option = None; - for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { - let peer = self.peer.read().await.clone(); - match peer { - Some(p) => match p.call_tool(request.clone()).await { - Ok(r) => { - self.mark_activity(); - return Ok(r); - } - Err(e) => { - let msg = e.to_string(); - if !is_transport_error_msg(&msg) || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 - { - return Err(McpError::internal_error(msg, None)); - } - tracing::warn!( - "call_tool('{}') attempt {}/{} failed (transport): {} — forcing reconnect", - request.name, - attempt + 1, - PROXY_MAX_RETRY_ATTEMPTS, - msg - ); - last_err = Some(msg); - self.force_reconnect().await; - } - }, - None => { - // Empty peer slot: either a deliberate idle-close or a real - // outage. `try_on_demand_connect` asks the main loop to - // connect *now* rather than waiting for the failure-path - // reconnect cadence to notice, bounded so we still fall back - // to the ordinary retry/backoff below if it doesn't land. - if attempt == 0 && self.try_on_demand_connect().await { - continue; - } - if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { - tokio::time::sleep(std::time::Duration::from_millis( - PROXY_RETRY_BACKOFF_MS, - )) - .await; - continue; - } - return Err(McpError::internal_error( - "codesearch serve is reconnecting — please retry in a moment".to_string(), - None, - )); - } - } - } - Err(McpError::internal_error( - last_err.unwrap_or_else(|| "transport error after retries".to_string()), - None, - )) - } -} - -/// Read model short-name and dimensions from a database's `metadata.json`. -/// Returns `(model_name, dimensions)`, defaulting to `("unknown", DEFAULT_EMBEDDING_DIMENSIONS)`. -fn read_model_metadata(db_path: &Path) -> (String, usize) { - let metadata_path = db_path.join("metadata.json"); - if let Ok(content) = std::fs::read_to_string(&metadata_path) { - if let Ok(json) = serde_json::from_str::(&content) { - let model_name = json - .get("model_short_name") - .and_then(|v| v.as_str()) - .unwrap_or("unknown") - .to_string(); - let dims = json.get("dimensions").and_then(|v| v.as_u64()).unwrap_or(0) as usize; - // If metadata has explicit dimensions, use those; otherwise infer from model name. - let dims = if dims > 0 { - dims - } else { - ModelType::parse(&model_name) - .map(|m| m.dimensions()) - .unwrap_or(crate::constants::DEFAULT_EMBEDDING_DIMENSIONS) - }; - return (model_name, dims); - } - } - ( - "unknown".to_string(), - crate::constants::DEFAULT_EMBEDDING_DIMENSIONS, - ) -} - -/// Read chunk/file counts from metadata.json (written after each indexing operation). -/// Returns `(total_chunks, total_files)` defaulting to `(0, 0)`. -/// -/// When metadata.json reports `total_chunks == 0` but the LMDB database exists, -/// falls back to opening the store read-only and counting live chunks. -/// This catches the case where a metadata writer clobbered the stats fields -/// (see `merge_metadata_atomic` for the definitive fix). The fallback is lazy — -/// only triggered when metadata reports zero — so it does not unnecessarily -/// open databases for repos that already have correct metadata. -fn read_metadata_stats(db_path: &Path) -> (usize, usize) { - let metadata_path = db_path.join("metadata.json"); - if let Ok(content) = std::fs::read_to_string(&metadata_path) { - if let Ok(json) = serde_json::from_str::(&content) { - let total_chunks = json - .get("total_chunks") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - let total_files = json - .get("total_files") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - - if total_chunks > 0 { - return (total_chunks, total_files); - } - - // Metadata says 0 chunks — try live LMDB count as fallback. - // This is safe in serve context: `read_metadata_stats` is only called - // for repos NOT yet opened in SharedStores (opened repos use vs.stats() - // directly), so no double-open risk. - if let Some((live_chunks, live_files)) = live_chunk_count(db_path) { - tracing::info!( - "metadata.json reports 0 chunks for {}, but LMDB has {} chunks / {} files — using live count", - db_path.display(), live_chunks, live_files - ); - return (live_chunks, live_files); - } - - return (total_chunks, total_files); - } - } - (0, 0) -} - -/// Open the LMDB read-only and count chunks/files. -/// Returns `None` if the database cannot be opened (missing, corrupt, or -/// already locked by another handle). -/// -/// # Safety (LMDB double-open) -/// -/// This function is only called when `get_opened_stores(alias)` returned `None`, -/// meaning no `SharedStores` handle exists for this repo. There is a theoretical -/// race window between that check and this `open_readonly` call where another task -/// could open the repo via `get_or_open_stores`. In practice this is safe because: -/// 1. The tokio runtime uses a single thread for non-spawned futures. -/// 2. Even if the race occurs, `open_readonly` returns `Err` (TrackedEnv blocks it), -/// and we return `None` — no crash, no corruption. -fn live_chunk_count(db_path: &Path) -> Option<(usize, usize)> { - let (model_name, dims) = read_model_metadata(db_path); - if model_name == "unknown" { + if !has_backed_source { return None; } - match VectorStore::open_readonly(db_path, dims) { - Ok(store) => match store.stats() { - Ok(stats) if stats.total_chunks > 0 => Some((stats.total_chunks, stats.total_files)), - Ok(_) => None, - Err(e) => { - tracing::debug!( - "live_chunk_count: stats() failed for {}: {}", - db_path.display(), - e - ); - None - } - }, - Err(e) => { - tracing::debug!( - "live_chunk_count: open_readonly failed for {}: {}", - db_path.display(), - e - ); - None - } - } + let backend = [ + crate::constants::LANG_CSHARP, + crate::constants::LANG_TYPESCRIPT, + ] + .into_iter() + .find(|lang| { + registry + .get(lang) + .is_some_and(|indexer| indexer.is_available()) + })?; + Some(format!( + "lexical text matching — hits may be docs/comments rather than code references; \ + for precise SCIP call-sites use find_impact (symbol_name='{symbol}', project=...) \ + (backend: {backend})" + )) } -/// RRF score threshold below which results are considered low-confidence. -/// When the top result's RRF score falls below this, the response includes -/// a `low_confidence` flag and a `suggested_tool` hint. -const LOW_CONFIDENCE_THRESHOLD: f32 = 0.02; - -/// Chunk kinds that represent symbol definitions (not usages/comments/etc.) -const DEFINITION_KINDS: &[&str] = &[ - "Function", - "Class", - "Method", - "Struct", - "Trait", - "Enum", - "TypeAlias", - "Interface", -]; - /// Codesearch MCP service pub struct CodesearchService { #[allow(dead_code)] @@ -688,8 +148,10 @@ pub struct CodesearchService { project_path: PathBuf, model_type: ModelType, dimensions: usize, - // Lazily initialized on first search - embedding_service: Arc>>, + // Lazily initialized on first search. A per-model pool: serve mode is + // multi-repo and each index records the model it was built with, so the + // query model is resolved per target repo (see `query_model`). + embedding_pool: Arc, // Shared stores for concurrent access (optional - only set when running with IndexManager) shared_stores: Option>, // Serve-mode state (set when running inside `codesearch serve`) @@ -734,43 +196,18 @@ impl Drop for CodesearchService { } } -// === Multi-store fan-out traits === - -/// Trait for types that have a chunk ID (used for deduplication in group fan-out). -trait HasChunkId { - fn chunk_id(&self) -> u32; -} - -/// Trait for types that have a relevance score (used for sorting in group fan-out). -trait HasScore { - fn score(&self) -> f32; -} - -impl HasChunkId for crate::vectordb::SearchResult { - fn chunk_id(&self) -> u32 { - self.id - } -} - -impl HasScore for crate::vectordb::SearchResult { - fn score(&self) -> f32 { - self.score - } -} - -impl HasChunkId for crate::fts::FtsResult { - fn chunk_id(&self) -> u32 { - self.chunk_id - } -} - -impl HasScore for crate::fts::FtsResult { - fn score(&self) -> f32 { - self.score - } +/// Outcome of resolving the embedding model for a query target. +/// +/// See [`CodesearchService::resolve_query_model`]. +pub(crate) struct QueryModel { + /// The model the query must be embedded with. + pub model: ModelType, + /// A caller-facing warning, set when the target index records no model and + /// the built-in default was assumed. `None` when the model was recorded or + /// the query is scope-free. + pub assumed_warning: Option, } -// === Simple Glob Matcher === // v1: supports prefix/suffix patterns with `*` and `**` only. /// Merge exact FTS results into the main result set, deduplicating by chunk_id /// and keeping the max score for duplicates. @@ -970,112 +407,6 @@ fn simple_glob_match_single_star(pattern: &str, path: &str) -> bool { true } -fn normalize_tool_path(path: &str, project_root: &Path) -> String { - let p = Path::new(path); - let resolved = if p.is_absolute() { - p.to_path_buf() - } else { - project_root.join(p) - }; - crate::cache::normalize_path_str(resolved.to_string_lossy().as_ref()) -} - -/// Strip a project-alias prefix from a tool path. -/// -/// In serve mode, tools like explore receive `target = "ALIAS/src/foo.rs"` with -/// `project = "ALIAS"`. The alias prefix must be stripped before calling -/// `chunks_for_file`, which expects a path relative to the project root. -fn strip_alias_prefix(path: &str, alias: Option<&String>) -> String { - if let Some(a) = alias { - let prefix = format!("{}/", a); - match path.strip_prefix(&prefix) { - Some(rest) => rest.to_string(), - None => path.to_string(), - } - } else { - path.to_string() - } -} - -/// Prefix a result path with its repo alias for group queries, normalizing -/// Windows backslashes to forward slashes in the process. When `alias` is -/// None or empty, the path is still normalized (useful for stdio mode). -pub(crate) fn prefix_path_with_alias( - path: &str, - alias: Option<&str>, - project_root: &str, -) -> String { - let normalized = crate::cache::normalize_path_str(path); - let normalized_root = crate::cache::normalize_path_str(project_root) - .trim_end_matches('/') - .to_string(); - match normalized.strip_prefix(&normalized_root) { - Some(rest) => { - let relative = rest.trim_start_matches('/'); - match alias { - Some(a) if !a.is_empty() => format!("{}/{}", a, relative), - _ => relative.to_string(), - } - } - None => normalized, - } -} - -/// Prefix a result path with the matching repo alias from a set of aliases and their roots. -/// Used by handlers that have alias/root info but not a full `MultiStoreContext`. -fn prefix_path_multi( - path: &str, - aliases: &[String], - alias_roots: &std::collections::HashMap, -) -> String { - let normalized = crate::cache::normalize_path_str(path); - for alias in aliases { - if let Some(root) = alias_roots.get(alias) { - if normalized.starts_with(root.as_str()) { - return prefix_path_with_alias(path, Some(alias), root); - } - } - } - normalized -} - -/// Pick the project root to relativise a result path against for a `filter_path` -/// prefix match, so `filter_path` is interpreted **relative to the repo root** -/// in every routing mode: -/// - serve single-project routing → the routed alias's root (`alias_roots[alias]`); -/// - serve multi/group → the longest alias root the (absolute) path lives under; -/// - stdio single-repo (no alias roots) → the service's own `project_path` -/// (`fallback_root`). -/// -/// Before this, the filter always used the service's `project_path`, which for a -/// serve-routed project is NOT the routed repo's root — so the absolute stored -/// path never relativised and every hit was dropped. The federated paths solve -/// the same class of bug client-side (see `retain_by_filter_path`); this covers -/// the local (non-federated) serve/multi case. -fn pick_filter_root( - path: &str, - project_alias: Option<&str>, - alias_roots: &std::collections::HashMap, - fallback_root: &str, -) -> String { - if let Some(alias) = project_alias { - if let Some(root) = alias_roots.get(alias) { - return root.clone(); - } - } - if !alias_roots.is_empty() { - let normalized = crate::cache::normalize_path_str(path); - if let Some(root) = alias_roots - .values() - .filter(|r| normalized.starts_with(r.as_str())) - .max_by_key(|r| r.len()) - { - return root.clone(); - } - } - fallback_root.to_string() -} - fn is_import_kind(kind: &str) -> bool { matches!(kind, "Import" | "Use" | "Require" | "Include" | "Imports") } @@ -1500,424 +831,38 @@ fn parse_import_lines(content: &str, start_line: usize) -> Vec { items } -// === Multi-Store Routing Context === - -/// Pre-computed routing context for a tool handler. -/// -/// Created by `CodesearchService::resolve_routing()`, this struct encapsulates -/// all the decisions a handler needs: which store to use, whether to fan out, -/// and whether to call `ensure_database_exists()`. -/// Outcome of a fan-out read across several repos. -/// -/// Exists so an empty `results` is never ambiguous. A group query that hits a -/// broken store used to come back as a successful search with zero hits, which -/// is the most misleading signal this system can emit — it reads as "the corpus -/// does not contain that", and it is what sent an earlier round of this -/// investigation chasing an indexing problem that did not exist. -#[must_use] -struct MultiReadOutcome { - /// Merged, deduplicated, score-sorted results from the stores that worked. - results: Vec, - /// `(alias, full error chain)` for every store that failed. Empty on a - /// clean run. - failures: Vec<(String, String)>, -} - -/// Decide the `status`/`status_message` pair for a multi-store -/// `status(kind="index"|"projects")` response. -/// -/// Pulled out of the handler so the four-way call — every store down, still -/// building, ready but one or more stores failed to report their stats, or -/// fully ready — is testable without opening a single store. `failed_count` -/// is checked before declaring "building" or "ready" precisely so a store -/// that came back `Err` cannot render identically to one that returned -/// healthy zero-valued stats. The all-failed case is checked first: a -/// correlated failure (e.g. every store hits the same read-only-snapshot or -/// disk-full condition at once) also has `total_chunks == 0`, and without -/// this ordering it fell through to "building" — byte-identical to a group -/// that simply has not been indexed yet, which is the exact indistinguishable -/// case this fix exists to close. See AGENTS.md's fan-out warnings-channel -/// rule. -fn index_status_summary( - total_repos: usize, - failed_count: usize, - total_chunks: usize, -) -> (String, String) { - if total_repos > 0 && failed_count >= total_repos { - ( - "error".to_string(), - format!( - "All {total_repos} repo(s) failed to report status — every store errored, see `warnings`." - ), - ) - } else if total_chunks == 0 { - ( - "building".to_string(), - format!( - "Index is being built across {total_repos} repo(s). Searches may fail until indexing completes." - ), - ) - } else if failed_count > 0 { - ( - "ready".to_string(), - format!( - "Index is ready for searching across {} of {total_repos} repo(s) — {failed_count} store(s) failed to report status, see `warnings`.", - total_repos.saturating_sub(failed_count), - ), - ) - } else { - ( - "ready".to_string(), - format!("Index is ready for searching across {total_repos} repo(s)."), - ) - } -} - -/// Turn a store's `stats()` result into the `(total_chunks, total_files, -/// error)` triple `list_projects` reports per repo. -/// -/// Pulled out of the handler, mirroring `index_status_summary` just above, so -/// the fix's actual claim — a `stats()` failure surfaces as `error: Some(..)` -/// with zero-valued counts, instead of silently rendering as a healthy-looking -/// empty repo — is unit-testable without opening a real `VectorStore` or -/// `ServeState`. The two calls to `serve_state.repo_lock_status()` in -/// `list_projects` don't vary by outcome, so they stay in the handler; this -/// covers only the part that does. -fn repo_stats_from_result( - stats: anyhow::Result, -) -> (usize, usize, Option) { - match stats { - Ok(s) => (s.total_chunks, s.total_files, None), - Err(ref e) => (0, 0, Some(format!("stats unavailable: {e:#}"))), - } -} - -/// `repo_stats_from_result` plus recording the failure as a caller-facing -/// warning, in one call. -/// -/// `list_projects` used to inline `repo_stats_from_result` and then decide -/// separately whether to push a warning — two steps a future edit could -/// silently pull apart (drop the second one, keep the first) without -/// affecting `total_chunks`/`total_files` at all, so nothing would look -/// wrong at the call site. Folding both into one call means a regression -/// that drops the warning has to delete this call entirely, which also -/// deletes the counts — no longer a silent edit. This is also the seam a -/// test can drive without opening a real `VectorStore`/`ServeState`: it -/// exercises the exact composition `list_projects` calls, not a -/// re-implementation of it. -fn record_stats_or_warn( - stats: anyhow::Result, - alias: &str, - warnings: &mut Vec, -) -> (usize, usize, Option) { - let (total_chunks, total_files, error) = repo_stats_from_result(stats); - if let Some(ref msg) = error { - push_store_warning(warnings, &store_warning(alias, "stats", msg)); - } - (total_chunks, total_files, error) -} - -/// Record a per-store failure as a caller-facing warning, once per store. +/// Whether a failed shared vector-store read may open a second `VectorStore` on `db_path`. /// -/// Resolution loops run per hit, so a single broken store would otherwise emit -/// one identical warning per result; the caller wants to know *that* the repo -/// is down, not how many times it noticed. -fn note_store_failure( - warnings: &mut Vec, - aliases: &[String], - idx: usize, - what: &str, - err: &anyhow::Error, -) { - let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); - push_store_warning(warnings, &store_warning(alias, what, &format!("{err:#}"))); +/// Stdio MCP (`--mode local`) and HTTP serve both hold a live `SharedStores` on +/// the same LMDB path. A second open is rejected by `TrackedEnv`. Gating only on +/// `serve_state` left stdio (shared stores, no serve state) on the fallback path. +pub(crate) fn allow_vector_store_second_open(has_shared_stores: bool) -> bool { + !has_shared_stores } -/// The one place a per-store warning line is formatted. Two copies used to -/// exist and could drift; a caller matching on this text would then silently -/// stop matching half of them. -fn store_warning(alias: &str, what: &str, err: &str) -> String { - format!("repo '{alias}' {what} failed: {err}") -} +// === Tool Router Implementation === -/// Append a warning unless it is already present, logging it once. -fn push_store_warning(warnings: &mut Vec, msg: &str) { - if !warnings.iter().any(|w| w == msg) { - tracing::error!("MCP: {}", msg); - warnings.push(msg.to_string()); +#[tool_router(allow_empty)] // ctors only; real tools merge in via merged_tool_router() +impl CodesearchService { + /// Create a new CodesearchService (standalone mode - opens its own VectorStore) + #[allow(dead_code)] // Reserved for standalone MCP server mode + pub fn new(requested_path: Option) -> Result { + Self::new_with_stores(requested_path, None) } -} -/// The single exit for a handler that returns a list of items plus a warnings -/// channel. -/// -/// Five handlers previously read their channel ONLY on the empty path, so a -/// partially-failed group returned a plausible-looking short list with no -/// signal at all - the same false negative as an empty result, just harder to -/// notice. Routing every exit through here means the channel is carried -/// whether the list is empty or not, and there is no per-handler discipline -/// left to forget. -/// -/// A healthy call is byte-identical to the previous behaviour (a bare JSON -/// array), so this is backward compatible. -fn respond_with_items( - items: &[T], - warnings: &[String], - empty_message: impl FnOnce() -> String, -) -> Result { - if items.is_empty() { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result(empty_message(), warnings), - )])); - } - if !warnings.is_empty() { - let payload = serde_json::json!({ "results": items, "warnings": warnings }); - return Ok(CallToolResult::success(vec![Content::text( - payload.to_string(), - )])); - } - let json = serde_json::to_string(items).unwrap_or_else(|_| "[]".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) -} + /// Create a new CodesearchService with shared stores (for use with IndexManager) + pub fn new_with_stores( + requested_path: Option, + shared_stores: Option>, + ) -> Result { + // Find the best database to use + let db_info = find_best_database(requested_path.as_deref())?; -/// The object-shaped sibling of `respond_with_items`: one exit for handlers that -/// return a single struct rather than a list. -/// -/// A `warnings` *field* on the response struct was the obvious fix and is the -/// weaker one — the handler is still free to populate it with `None`, and a test -/// that builds the struct itself cannot see that happen. Review round 8 proved -/// it: the round-7 defect was reintroduced at the `get_chunk` success path and -/// all 630 tests still passed. -/// -/// **This is an improvement, not a guarantee.** Round 9 measured the difference: -/// passing `&[]` here is exactly as writable as `warnings: None` was, the suite -/// still cannot see it, and no lint fires (the channel stays "used" by the -/// ambiguous path). What it actually buys is narrower and real — no optional -/// field whose absence is invisible, no future construction site that can zero -/// it, and an audit that collapses from "check every response struct" to "check -/// the call sites of two functions", which is grep-answerable. The channel can -/// no longer be *forgotten*, only actively discarded. -/// -/// Healthy path serializes the struct directly, so its key order and bytes are -/// unchanged. `serde_json::Map` is a `BTreeMap` here (no `preserve_order` -/// feature), so round-tripping through `to_value` would silently re-sort the -/// keys — which is only acceptable on the warning path, where the shape is new -/// anyway. -fn respond_with_object( - value: &T, - warnings: &[String], -) -> Result { - if !warnings.is_empty() { - if let Ok(mut v) = serde_json::to_value(value) { - if let Some(obj) = v.as_object_mut() { - obj.insert("warnings".to_string(), serde_json::json!(warnings)); - return Ok(CallToolResult::success(vec![Content::text(v.to_string())])); - } - } - } - let json = serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) -} - -/// Build the `ambiguous_chunk_id` payload for `get_chunk`. -/// -/// `candidate_projects` reads as the complete set of repos holding this -/// chunk_id, so a store that failed to answer must be declared: the repo the -/// caller actually wants may be the one missing from the list. Extracted so the -/// "is this list complete?" decision is testable without standing up stores. -/// -/// `warnings` is *inserted* rather than emitted as `null`, so the healthy-path -/// shape is byte-identical to before — matching `skip_serializing_if` on every -/// other warnings-carrying response. -fn ambiguous_chunk_payload( - chunk_id: u32, - candidate_projects: &[&str], - warnings: &[String], -) -> serde_json::Value { - let mut message = - format!("chunk_id {chunk_id} exists in multiple repositories. Specify which one."); - if !warnings.is_empty() { - message.push_str(" The candidate list is incomplete — see `warnings`."); - } - let mut payload = serde_json::json!({ - "error_code": "ambiguous_chunk_id", - "message": message, - "candidate_projects": candidate_projects, - "hint_for_agent": "The chunk_id collision is a known limitation of multi-repo mode. Re-run get_chunk with one of the candidate_projects, or use search to identify the correct repository first." - }); - if !warnings.is_empty() { - if let Some(obj) = payload.as_object_mut() { - obj.insert("warnings".to_string(), serde_json::json!(warnings)); - } - } - payload -} - -/// Decide whether to keep the "try another tool" hint. -/// -/// A weak or empty result caused by a store that is DOWN is not a reason to -/// retry with a different tool — that just sends the agent back at the same -/// broken store. Extracted from `build_semantic_response` so the decision is -/// testable without standing up a service. -fn retry_hint(suggested: Option, warnings: &Option>) -> Option { - // `is_some()` alone is wrong: an empty `Some(vec![])` means nothing failed, - // and suppressing a legitimate hint on it would be a silent regression the - // moment a caller constructs the warnings vec eagerly. - if warnings.as_ref().is_some_and(|w| !w.is_empty()) { - return None; - } - suggested -} - -/// Qualify a "nothing found" message when a store in scope actually failed. -/// -/// This is the defect that keeps coming back in a new handler: "No definition -/// found — the symbol may not be indexed" is a *diagnosis*, and it is flatly -/// wrong when the store never answered. An agent acts on it by giving up or by -/// re-indexing something that was never broken. -fn qualify_empty_result(message: String, warnings: &[String]) -> String { - if warnings.is_empty() { - return message; - } - format!( - "{message}\n\nWARNING: this result is not trustworthy — {count} store(s) in \ - scope failed, so \"not found\" may mean \"not searched\":\n{detail}", - count = warnings.len(), - detail = warnings.join("\n") - ) -} - -// Hand-written rather than derived: `derive(Default)` would demand `R: Default`, -// which the result types do not implement and do not need to. -impl Default for MultiReadOutcome { - fn default() -> Self { - Self { - results: Vec::new(), - failures: Vec::new(), - } - } -} - -impl MultiReadOutcome { - /// Render failures as caller-facing warning lines. - fn warnings(&self, what: &str) -> Vec { - self.failures - .iter() - .map(|(alias, err)| store_warning(alias, what, err)) - .collect() - } - - /// Take the results, routing any failures into `warnings` on the way out. - /// - /// Deliberately the only ergonomic way to get at `results`: reaching for - /// the field directly and dropping `failures` is `unwrap_or_default()` - /// under a new name, and that is the bug this whole type exists to stop. - fn into_results(self, warnings: &mut Vec, what: &str) -> Vec { - for (alias, err) in &self.failures { - push_store_warning(warnings, &store_warning(alias, what, err)); - } - self.results - } -} - -struct MultiStoreContext { - /// Single-store override (set when exactly 1 repo resolved, or None). - /// Pass to `with_*_store_read_for()` methods. - stores: Option>, - /// Multi-store vec for fan-out (set when 2+ repos resolved, or None). - /// Use `if let Some(ref sv) = ctx.stores_vec { ... }` for the multi-store path. - stores_vec: Option>>, - /// Alias for each store in `stores_vec` (parallel with stores_vec). - /// Used for path prefixing and per-alias dedup. - store_aliases: Option>, - /// Alias for single-project routing (set when project= is given). - project_alias: Option, - /// Normalized project root for each alias (alias → root path). - /// Used by `prefix_path` to strip absolute paths and add alias prefix. - alias_roots: std::collections::HashMap, - /// True when `stores_vec` has 2+ entries (group fan-out). - is_multi: bool, - /// True when no serve-state stores resolved and local DB should be checked. - needs_local_db: bool, -} - -impl MultiStoreContext { - /// Aliases parallel to `stores_vec`, or an empty slice when absent. - /// - /// Every fan-out that reports a per-store failure needs this, and hand-rolling - /// `let empty = Vec::new(); ...unwrap_or(&empty)` at each site produced four - /// copies of the same two lines — and one handler where the binding was out - /// of scope, which is how a silent store read survived a round of review. - fn aliases(&self) -> &[String] { - self.store_aliases.as_deref().unwrap_or(&[]) - } - - /// Prefix a result path with its owning alias for multi-repo identification. - /// - /// Three dispatch modes: - /// - Single-project (`project_alias = Some(...)`): prefix with that alias. - /// - Group (`store_aliases = Some([...])`): detect alias by prefix-matching - /// the path against known project roots in `alias_roots`. - /// - Stdio / no alias info: normalize only, no prefix. - /// - /// Emits a `tracing::debug!` event when an expected alias cannot be resolved. - /// That usually indicates a config mismatch or a path from an unregistered source — - /// the path is still normalized and returned, but diagnosis is easier with the log. - fn prefix_result_path(&self, path: &str) -> String { - if let Some(ref alias) = self.project_alias { - if let Some(root) = self.alias_roots.get(alias) { - return prefix_path_with_alias(path, Some(alias), root); - } - tracing::debug!( - target: "codesearch::mcp::path_prefix", - alias = %alias, - path = %path, - "project_alias has no entry in alias_roots" - ); - } - if let Some(ref aliases) = self.store_aliases { - let normalized = crate::cache::normalize_path_str(path); - for alias in aliases { - if let Some(root) = self.alias_roots.get(alias) { - if normalized.starts_with(root.as_str()) { - return prefix_path_with_alias(path, Some(alias), root); - } - } - } - tracing::debug!( - target: "codesearch::mcp::path_prefix", - aliases = ?aliases, - path = %path, - "no alias root matched path in group mode" - ); - } - crate::cache::normalize_path_str(path) - } -} - -// === Tool Router Implementation === - -#[tool_router] -impl CodesearchService { - /// Create a new CodesearchService (standalone mode - opens its own VectorStore) - #[allow(dead_code)] // Reserved for standalone MCP server mode - pub fn new(requested_path: Option) -> Result { - Self::new_with_stores(requested_path, None) - } - - /// Create a new CodesearchService with shared stores (for use with IndexManager) - pub fn new_with_stores( - requested_path: Option, - shared_stores: Option>, - ) -> Result { - // Find the best database to use - let db_info = find_best_database(requested_path.as_deref())?; - - if db_info.is_none() { - return Err(anyhow::anyhow!( - "No database found in current directory, parent directories, or globally tracked repositories. \ - Run 'codesearch index' first to index the codebase." - )); + if db_info.is_none() { + return Err(anyhow::anyhow!( + "No database found in current directory, parent directories, or globally tracked repositories. \ + Run 'codesearch index' first to index the codebase." + )); } let db_info = db_info.unwrap(); @@ -1948,12 +893,14 @@ impl CodesearchService { }; Ok(Self { - tool_router: Self::tool_router(), + tool_router: Self::merged_tool_router(), db_path, project_path, model_type, dimensions, - embedding_service: Arc::new(Mutex::new(None)), + embedding_pool: Arc::new(EmbeddingServicePool::new( + crate::constants::get_global_models_cache_dir().ok(), + )), shared_stores, serve_state: None, symbol_registry: Arc::new(SymbolIndexerRegistry::new()), @@ -1967,13 +914,19 @@ impl CodesearchService { /// it routes requests to the repo identified by `project`/`group`. pub(crate) fn new_for_serve(serve_state: Arc) -> Result { let symbol_registry = serve_state.symbol_registry(); + // Seed the service with the serve-wide default model (`serve --model`), + // the same value `POST /repos` stamps into a newly created index, so the + // scope-free status summary reports it. It is deliberately NOT the query + // fallback for a repo whose metadata records no model — that resolves to + // the built-in default, with a warning. See `resolve_query_model`. + let model_type = serve_state.default_model().unwrap_or_default(); Ok(Self { - tool_router: Self::tool_router(), + tool_router: Self::merged_tool_router(), db_path: PathBuf::from("serve://multi-repo"), project_path: PathBuf::from("serve://multi-repo"), - model_type: ModelType::default(), - dimensions: crate::constants::DEFAULT_EMBEDDING_DIMENSIONS, - embedding_service: serve_state.embedding_service(), + model_type, + dimensions: model_type.dimensions(), + embedding_pool: serve_state.embedding_pool(), shared_stores: None, serve_state: Some(serve_state), symbol_registry, @@ -1992,17 +945,71 @@ impl CodesearchService { self.tracks_session = true; } - /// Get or initialize the embedding service - fn get_embedding_service(&self) -> Result>> { - let mut guard = self.embedding_service.lock().unwrap(); - if guard.is_none() { - let cache_dir = crate::constants::get_global_models_cache_dir()?; - *guard = Some(EmbeddingService::with_cache_dir( - self.model_type, - Some(&cache_dir), - )?); + /// Resolve the embedding model a query against `alias` must use. + /// + /// With a repo alias (`project=` / group member) the model is read from that + /// repo's index metadata — an index built with EmbeddingGemma must be + /// queried with EmbeddingGemma, not the 384-dim default. A repo whose + /// metadata records no model is queried with the built-in default, never the + /// serve-wide `--model` default (see [`Self::resolve_query_model`]). With no + /// alias this returns the service's own `model_type`: the local index + /// metadata in stdio mode, the serve default in serve mode (the scope-free + /// status summary). Prefer [`Self::resolve_query_model`] when the caller can + /// surface the unrecorded-model warning. + pub(crate) fn query_model(&self, alias: Option<&str>) -> ModelType { + self.resolve_query_model(alias).model + } + + /// Resolve the query model together with a warning when it had to be assumed. + /// + /// The model a query is embedded with must match the model the target index + /// was built with. When `alias`'s metadata records no model the model is + /// unknowable, so this assumes the BUILT-IN default: that is both the + /// historical 384-dim behaviour and the value every other reader assumes for + /// metadata without a `model_short_name`. It deliberately does NOT assume the + /// serve-wide `--model` default: that flag selects the model for newly + /// created indexes, and using it here would break a working legacy repo the + /// moment an operator set it (a 384-dim index queried with a 768-dim model + /// fails, and a same-dimension model degrades rankings silently). The + /// returned warning names the repo, the assumption, and the re-index command. + pub(crate) fn resolve_query_model(&self, alias: Option<&str>) -> QueryModel { + if let (Some(state), Some(alias)) = (self.serve_state.as_ref(), alias) { + if let Some(model) = state.model_for_alias(alias) { + return QueryModel { + model, + assumed_warning: None, + }; + } + let model = ModelType::default(); + let warning = format!( + "repo '{alias}' records no embedding model; queried with the built-in default '{}' ({} dims). If this repo was indexed with a different model, re-index it: codesearch index --force --model ", + model.short_name(), + model.dimensions() + ); + if state.mark_legacy_model_warned(alias) { + tracing::warn!("{}", warning); + } + return QueryModel { + model, + assumed_warning: Some(warning), + }; + } + QueryModel { + model: self.model_type, + assumed_warning: None, } - Ok(guard) + } + + /// Get (lazily initializing) the embedding service for `model`. + /// + /// The returned `Arc>` is per-model, so concurrent queries against + /// different models do not serialise on one global lock. Callers MUST pass + /// the model the target index was built with — see [`Self::query_model`]. + pub(crate) fn embedding_service_for( + &self, + model: ModelType, + ) -> Result>> { + self.embedding_pool.get(model) } /// Return the current MCP mode as a string for diagnostics. @@ -2268,30 +1275,19 @@ impl CodesearchService { Err(shared_err) => { tracing::error!("Shared vector store read failed: {:?}", shared_err); - // In serve mode, do NOT fall back to a standalone VectorStore — - // it would open a second LMDB handle on the same .codesearch.db, - // which LMDB rejects ("environment already opened with different options"). - if self.serve_state.is_some() { + // This branch already holds a live SharedStores (stdio and serve). + // Do not open a second VectorStore on the same db_path. + if !allow_vector_store_second_open(true) { return Err(shared_err.context( - "Shared vector store read failed in serve mode; \ + "Shared vector store read failed; \ standalone fallback disabled to prevent LMDB double-open", )); } } } - - // Standalone readonly fallback (non-serve mode only). - // In serve mode the guard above already returned. - if stores.readonly { - let ro_store = VectorStore::open_readonly(&self.db_path, self.dimensions) - .context("Error opening readonly database for read fallback")?; - return action(&ro_store) - .context("Error reading from readonly fallback vector store"); - } } - // Standalone fallback (non-serve mode only — CLI / stdio MCP). - // In serve mode, either Priority 1/2 succeeded or the guard above returned Err. + // No live shared store: true standalone CLI (not stdio MCP with SharedStores). let store = VectorStore::new(&self.db_path, self.dimensions) .context("Error opening database for read fallback")?; action(&store).context("Error reading from vector store") @@ -2327,8 +1323,11 @@ impl CodesearchService { /// Fan-out vector store read across multiple stores, merging results. /// - /// Runs `action` against each store and merges all results into a single vec, - /// deduplicating by (alias, chunk_id) (keeping highest score) and sorting by score descending. + /// Runs `action(alias, store)` against each store and merges all results into + /// a single vec, deduplicating by (alias, chunk_id) (keeping highest score) + /// and sorting by score descending. The `alias` is passed to the closure so + /// callers can select per-repo state — notably the query embedding for that + /// repo's own model (see `semantic_search_multi`). /// /// A per-store failure does NOT abort the fan-out — one broken repo should /// not blind a group query to the healthy ones — but it is reported back in @@ -2341,7 +1340,7 @@ impl CodesearchService { aliases: &[String], ) -> Result> where - F: FnMut(&VectorStore) -> anyhow::Result>, + F: FnMut(&str, &VectorStore) -> anyhow::Result>, R: Clone + HasChunkId + HasScore, { let mut failures: Vec<(String, String)> = Vec::new(); @@ -2352,7 +1351,7 @@ impl CodesearchService { for (idx, store_arc) in stores.iter().enumerate() { let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); let store = store_arc.vector_store.read().await; - match action(&store) { + match action(alias, &store) { Ok(results) => { for r in results { let key = (alias.to_string(), r.chunk_id()); @@ -2574,7 +1573,7 @@ impl CodesearchService { self.literal_search(Parameters(req)).await? } _ => { - return Ok(CallToolResult::success(vec![Content::text(format!( + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( "Unknown search mode '{}'. Use `semantic` or `literal`.", mode ))])); @@ -2746,7 +1745,7 @@ impl CodesearchService { let (peer_name, remote_alias, chunk_id) = match parse_federated_chunk_ref(chunk_ref) { Some(parts) => parts, None => { - return Ok(CallToolResult::success(vec![Content::text(format!( + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( "Invalid chunk_ref '{}': expected '/:'.", chunk_ref ))])); @@ -2757,7 +1756,7 @@ impl CodesearchService { Some(p) => p.clone(), None => { let known: Vec = cfg.remotes.keys().cloned().collect(); - return Ok(CallToolResult::success(vec![Content::text(format!( + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( "Unknown remote peer '{}' in chunk_ref '{}'. Known remotes: {}", peer_name, chunk_ref, @@ -2768,7 +1767,7 @@ impl CodesearchService { let client = match FederationClient::new() { Ok(c) => c, Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( "federation disabled (http client error): {e}" ))])); } @@ -2783,11 +1782,11 @@ impl CodesearchService { .get_chunk(&peer, remote_alias, chunk_id, context_lines) .await { - Outcome::Ok(value) => Ok(CallToolResult::success(vec![Content::text( + Outcome::Ok(value) => Ok(CallToolResult::success(vec![ContentBlock::text( value.to_string(), )])), Outcome::Unreachable(reason) => { - Ok(CallToolResult::success(vec![Content::text(format!( + Ok(CallToolResult::success(vec![ContentBlock::text(format!( "Could not fetch chunk from remote peer '{}': {}", peer_name, reason ))])) @@ -2819,4089 +1818,129 @@ impl CodesearchService { }, }; let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - CallToolResult::success(vec![Content::text(json)]) + CallToolResult::success(vec![ContentBlock::text(json)]) } +} - // ───────────────────────────────────────────────────────────────── - // Consolidated tools (the primary 5-tool surface) - // ───────────────────────────────────────────────────────────────── +// === Server Handler Implementation === - /// Unified search tool — dispatches to semantic or literal search based on `mode`. - #[tool( - description = "Unified code search. Set `mode` to choose the backend:\n\n- `semantic` (default): vector embeddings + BM25 FTS + exact-identifier boosting, fused with RRF. Best for conceptual queries, identifier lookups, and mixed natural-language + symbol queries.\n- `literal`: pure FTS, no embeddings. Fast and works without an embedding model. Sub-mode selection:\n * Queries with operators, brackets, or punctuation (`foo = null`, `Vec`, `return x;`, `a::b`) -> set `regex=true` and write the query as a regex. BM25 tokenizes on punctuation otherwise, producing noisy results.\n * Multi-word exact phrases -> set `phrase=true`.\n * Plain identifier lookups (`CodesearchService`) -> leave both false.\n\nFor semantic mode, optionally set `semantic_mode`: \"auto\" (default) | \"semantic\" | \"lexical\" | \"hybrid\".\nReturns metadata only by default (`compact=true`). Use `get_chunk` to read full code. Prefer `search(mode=\"literal\", regex=true)` over external grep/ripgrep for code patterns.\n\nIMPORTANT (multi-repo): always specify either `project` (single repo) or `group` (cross-repo). Omitting both in multi-repo mode returns a `scope_required` error with the list of available projects and groups. If the user has not indicated which repository to search, ask them to choose." - )] - async fn search( - &self, - Parameters(request): Parameters, - ) -> Result { - tracing::info!( - "📥 search(query={:?}, mode={:?}, project={:?}, group={:?})", - request.query, - request.mode, - request.project, - request.group, - ); - - // Federation: when the query targets a group that resolves to one or more - // remote peers, merge local + remote results (RRF-interleave) instead of - // searching local repos only. Only `group` federates; `project` stays - // local because project aliases are instance-local. - if let Some(group) = request.group.as_deref() { - let cfg = self.federation_config(); - if Self::group_has_remotes(&cfg, group) { - let remote_projects = cfg.group_remote_projects(group); - return self.federated_search(&request, &cfg, remote_projects).await; - } - } +/// Check if a chunk is a definition of the given symbol. +/// +/// Best-effort heuristic for v1: a chunk is considered a definition if: +/// 1. Its kind is a definition kind (Function, Struct, Class, etc.) +/// 2. Its signature starts with a common definition pattern containing the symbol name +/// +/// Limitation: this uses simple substring matching on the signature field. +/// False positives/negatives are possible for symbols that appear in signatures +/// of chunks that are not their definitions. +fn is_definition_chunk(kind: &str, signature: &Option, symbol: &str) -> bool { + // Only check definition kinds + if !DEFINITION_KINDS.contains(&kind) { + return false; + } - // Project-level federation (mounted remote project): a `project` of the - // form "/" transparently routes to that single peer's own - // `` project — a 1-to-1 passthrough, as if the index were local. - // Local repos ALWAYS win a name clash: only route remotely when the name - // does not resolve to a local project. - if let Some(proj) = request.project.as_deref() { - let cfg = self.federation_config(); - if cfg.resolve(proj).is_none() { - if let Some(crate::db_discovery::repos::Target::RemoteProject { - peer_name, - peer, - remote_alias, - }) = cfg.resolve_remote_project(proj) - { - return self - .federated_project_search(&request, peer_name, peer, remote_alias) - .await; - } - } - } + let sig = match signature { + Some(s) if !s.is_empty() => s, + _ => return false, + }; - let mode = request.mode.as_deref().unwrap_or("semantic").to_lowercase(); - match mode.as_str() { - "semantic" => { - // Delegate to the existing semantic_search implementation - let semantic_req = SemanticSearchRequest { - query: request.query, - limit: request.limit, - compact: request.compact, - filter_path: request.filter_path, - mode: request.semantic_mode, - project: request.project, - group: request.group, - }; - self.semantic_search(Parameters(semantic_req)).await - } - "literal" => { - // Delegate to the existing literal_search implementation - let literal_req = LiteralSearchRequest { - query: request.query, - regex: request.regex, - phrase: request.phrase, - limit: request.limit, - file_glob: request.file_glob, - language: request.language, - format: request.format, - project: request.project, - group: request.group, - }; - self.literal_search(Parameters(literal_req)).await - } - _ => Ok(CallToolResult::success(vec![Content::text(format!( - "Unknown search mode '{}'. Use `semantic` or `literal`.", - mode - ))])), - } - } + // Common definition prefixes across languages. + // Keep this allocation-free in hot paths by using &str prefixes and boundary checks. + const PREFIXES: &[&str] = &[ + "fn ", + "def ", + "class ", + "struct ", + "enum ", + "trait ", + "type ", + "interface ", + "impl ", + "pub fn ", + "pub async fn ", + "pub struct ", + "pub enum ", + "pub trait ", + "pub type ", + "async fn ", + "const ", + "static ", + ]; - /// Unified symbol navigation — dispatches based on `kind`. - #[tool( - description = "Unified symbol navigation. Set `kind` to choose the action:\n\n- `definition` (default): locate where a symbol is defined (function, class, struct, etc.)\n- `usages`: find all call-sites and references to a symbol (lexical/text-based; for IDE-precise call-graphs prefer `find_impact`)\n- `imports`: list all imports/dependencies declared in a file (set `symbol` to the file path)\n- `dependents`: find all files that import or depend on a module, file, or symbol\n\nFor `imports`, set `symbol` to a file path. For other kinds, `symbol` is the symbol name.\n\nIMPORTANT (multi-repo): always specify either `project` (single repo) or `group` (cross-repo). Omitting both in multi-repo mode returns a `scope_required` error with the list of available projects and groups. If the user has not indicated which repository to search, ask them to choose." - )] - async fn find( - &self, - Parameters(request): Parameters, - ) -> Result { - let kind = request - .kind - .as_deref() - .unwrap_or("definition") - .to_lowercase(); - tracing::info!( - "📥 find(symbol={:?}, kind={}, project={:?}, group={:?})", - request.symbol, - kind, - request.project, - request.group, - ); - match kind.as_str() { - "definition" => { - let def_req = FindDefinitionRequest { - symbol: request.symbol, - kind: request.definition_kind, - limit: request.limit, - project: request.project, - group: request.group, - }; - self.find_definition(Parameters(def_req)).await - } - "usages" => { - let usages_req = FindUsagesRequest { - symbol: request.symbol, - limit: request.limit, - project: request.project, - group: request.group, - }; - self.find_usages(Parameters(usages_req)).await - } - "imports" => { - let imports_req = FindImportsRequest { - path: request.symbol, - project: request.project, - group: request.group, - }; - self.find_imports(Parameters(imports_req)).await - } - "dependents" => { - let dep_req = FindDependentsRequest { - symbol_or_path: request.symbol, - limit: request.limit, - project: request.project, - group: request.group, - }; - self.find_dependents(Parameters(dep_req)).await - } - _ => Ok(CallToolResult::success(vec![Content::text(format!( - "Unknown find kind '{}'. Use `definition`, `usages`, `imports`, or `dependents`.", - kind - ))])), + let prefix_match = PREFIXES.iter().any(|prefix| { + if !sig.starts_with(prefix) { + return false; } - } - /// Unified exploration tool — dispatches based on `kind`. - #[tool( - description = "Unified code exploration. Set `kind` to choose the action:\n\n- `outline` (default): list all indexed top-level symbols in a file — kind, signature, and line range. Set `target` to a file path.\n- `similar`: find chunks semantically similar to a given chunk by its ID. Set `target` to the chunk_id (as string).\n\nIMPORTANT (multi-repo): always specify either `project` (single repo) or `group` (cross-repo). Omitting both in multi-repo mode returns a `scope_required` error with the list of available projects and groups. If the user has not indicated which repository to search, ask them to choose." - )] - async fn explore( - &self, - Parameters(request): Parameters, - ) -> Result { - let kind = request.kind.as_deref().unwrap_or("outline").to_lowercase(); - tracing::info!( - "📥 explore(target={:?}, kind={}, project={:?})", - request.target, - kind, - request.project, - ); - match kind.as_str() { - "outline" => { - let outline_req = FileOutlineRequest { - path: request.target, - project: request.project, - group: request.group, - }; - self.file_outline(Parameters(outline_req)).await - } - "similar" => { - let chunk_id = match request.target.parse::() { - Ok(id) => id, - Err(_) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "For similar mode, `target` must be a numeric chunk_id, got: '{}'", - request.target - ))])); - } - }; - let similar_req = SimilarChunksRequest { - chunk_id, - limit: request.limit, - project: request.project, - group: request.group, - }; - self.similar_chunks(Parameters(similar_req)).await - } - _ => Ok(CallToolResult::success(vec![Content::text(format!( - "Unknown explore kind '{}'. Use `outline` or `similar`.", - kind - ))])), + let rest = &sig[prefix.len()..]; + if !rest.starts_with(symbol) { + return false; } - } - /// Unified status tool — dispatches based on `kind`. - #[tool( - description = "Unified status/info tool. Set `kind` to choose the action:\n\n- `index` (default): get the status of the local search index (model info, chunk count, readiness)\n- `projects`: list all registered projects/repositories, groups, and their index status" - )] - async fn status( - &self, - Parameters(request): Parameters, - ) -> Result { - let kind = request.kind.as_deref().unwrap_or("index").to_lowercase(); - tracing::info!("📥 status(kind={})", kind); - match kind.as_str() { - "index" => self.index_status_impl(request.project, request.group).await, - "projects" => self.list_projects().await, - _ => Ok(CallToolResult::success(vec![Content::text(format!( - "Unknown status kind '{}'. Use `index` or `projects`.", - kind - ))])), - } - } + let next = rest[symbol.len()..].chars().next(); + matches!(next, None | Some('(' | '<' | ':' | ' ' | '\t')) + }); - // ───────────────────────────────────────────────────────────────── - // Internal implementations (called by consolidated tools above) - // ───────────────────────────────────────────────────────────────── + if prefix_match { + return true; + } - /// Internal: semantic/hybrid search implementation used by `search(mode="semantic")`. - async fn semantic_search( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing (multi-store for group fan-out) - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "search") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; + // Fallback for languages with verbose signatures (C#, Java): + // signatures include access modifiers and return types before the symbol name, + // e.g. "public async Task UploadFileAsync(...)" or "protected override void Update(...)". + // Search for the symbol as a whole word anywhere in the signature. + contains_symbol_as_word(sig, symbol) +} - let limit = request.limit.unwrap_or(10); - let compact = request.compact.unwrap_or(true); - let mode = request.mode.as_deref().unwrap_or("auto"); - let identifiers = detect_identifiers(&request.query); - let has_identifiers = !identifiers.is_empty(); - - tracing::debug!( - "MCP semantic_search: query='{}', limit={}, compact={}, mode='{}', multi={}", - request.query, - limit, - compact, - mode, - ctx.is_multi - ); - - // Ensure database exists (skip if serve-mode with routed stores) - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); +/// Check whether `symbol` appears as a whole word in `sig`. +/// A word boundary requires the character before to be a space/tab (or start-of-string) +/// and the character after to be `(`, `<`, `:`, space, tab, or end-of-string. +/// This is intentionally conservative to avoid matching parameter type names. +fn contains_symbol_as_word(sig: &str, symbol: &str) -> bool { + let sig_bytes = sig.as_bytes(); + let sym_len = symbol.len(); + let mut start = 0usize; + while start + sym_len <= sig.len() { + if let Some(rel) = sig[start..].find(symbol) { + let abs = start + rel; + let before_ok = abs == 0 + || matches!( + sig_bytes.get(abs - 1), + Some(&b' ') | Some(&b'\t') | Some(&b'\n') + ); + let after_char = sig[abs + sym_len..].chars().next(); + let after_ok = matches!(after_char, None | Some('(' | '<' | ':' | ' ' | '\t')); + if before_ok && after_ok { + return true; } + start = abs + 1; + } else { + break; } + } + false +} - // === Multi-store group fan-out === - if ctx.is_multi { - return self - .semantic_search_multi( - &request, - &identifiers, - limit, - compact, - ctx.stores_vec.unwrap(), - ctx.store_aliases.as_ref().unwrap(), - &ctx.alias_roots, - ) - .await; - } - - // === Mode: "lexical" — FTS only, no embedding === - if mode == "lexical" { - tracing::debug!("MCP: mode=lexical — skipping embedding service"); - return self - .semantic_search_lexical( - &request, - &identifiers, - limit, - compact, - ctx.stores, - ctx.project_alias.as_deref(), - &ctx.alias_roots, - ) - .await; - } - - // === Modes: "semantic", "hybrid", "auto" — require embedding === - let query_embedding = { - let mut service_guard = match self.get_embedding_service() { - Ok(g) => g, - Err(e) => { - tracing::error!("MCP: Failed to get embedding service: {:?}", e); - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error initializing embedding service: {e:#}" - ))])); - } - }; +// ════════════════════════════════════════════════════════════════ +// REST API handlers (federation-friendly HTTP+JSON mirror of MCP tools). +// +// Expose the same logic over plain HTTP so a remote codesearch serve can be +// queried for federation WITHOUT an MCP session. Each handler constructs a +// throwaway `CodesearchService` bound to the live `ServeState`, invokes the +// existing `#[tool]` method, and returns the tool's JSON payload unwrapped +// from `CallToolResult`. Protected by serve's `require_auth_for_network` +// layer (same as /status, /mcp) — no separate auth code needed. +// ════════════════════════════════════════════════════════════════ +use axum::extract::{Path as AxumPath, Query as AxumQuery, State as AxumState}; +use axum::http::StatusCode; +use axum::response::Json as AxumJson; - let service = service_guard.as_mut().unwrap(); - tracing::debug!("MCP: Embedding query..."); - match service.embed_query(&request.query) { - Ok(e) => e, - Err(e) => { - tracing::error!("MCP: Failed to embed query: {:?}", e); - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error embedding query: {e:#}" - ))])); - } - } - }; - - // Failures on this single-store path. The group fan-out has carried a - // warnings channel since the read-only incident; without the same thing - // here, `project=` — the form an agent uses most — still reports - // a broken store as an ordinary empty result. - let mut single_warnings: Vec = Vec::new(); - - // Search vector store - let vector_results = match self - .with_vector_store_read_for( - |store| { - store - .search(&query_embedding, limit * 5) - .context("Error searching vector store") - }, - ctx.stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - tracing::error!("MCP: Search failed: {:?}", e); - // Only "semantic" has no second backend to fall back on. In - // hybrid/auto the FTS half can still answer, so hard-failing - // here would throw away good results — the same mistake this - // branch already fixed once in the group fan-out. - // - // `{:#}` renders the whole anyhow chain. With plain `{}` the - // caller only ever saw the outermost `.context(...)` wrapper - // ("Error reading from project-routed vector store"), which - // hides the actual fault and makes remote diagnosis guesswork. - if mode == "semantic" { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching vector store: {:#}", - e - ))])); - } - single_warnings.push(format!("vector search failed: {e:#}")); - Vec::new() - } - }; - - tracing::debug!("MCP: Found {} vector results", vector_results.len()); - - // === Mode: "semantic" — vector only, skip FTS fusion === - if mode == "semantic" { - tracing::debug!("MCP: mode=semantic — using vector results only"); - let fused = vector_only(&vector_results); - - let chunk_to_result: std::collections::HashMap = - vector_results.iter().map(|r| (r.id, r)).collect(); - - let mut results: Vec = Vec::new(); - for f in fused.into_iter().take(limit) { - if let Some(result) = chunk_to_result.get(&f.chunk_id) { - let mut r = (*result).clone(); - r.score = f.rrf_score; - results.push(r); - } - } - return self.build_semantic_response( - results, - &request, - compact, - has_identifiers, - ctx.project_alias.as_deref(), - &ctx.alias_roots, - &single_warnings, - ); - } - - // === Modes: "hybrid" | "auto" — full hybrid search === - let structural_intent = detect_structural_intent(&request.query); - let (vector_k, fts_k) = adapt_rrf_k(&request.query); - - tracing::debug!( - "MCP: Query analysis - identifiers: {:?}, structural_intent: {:?}, rrf_k: ({}, {})", - identifiers, - structural_intent, - vector_k, - fts_k - ); - - // Perform FTS search and fusion - let mut results = match self - .with_fts_store_read_for( - |fts_store| { - let fts_results = fts_store - .search(&request.query, limit * 5, structural_intent) - .context("Error searching FTS store")?; - - let fused = if identifiers.is_empty() { - rrf_fusion(&vector_results, &fts_results, vector_k as f32) - } else { - let mut all_exact: Vec = Vec::new(); - for ident in &identifiers { - if let Ok(exact) = - fts_store.search_exact(ident, limit * 3, structural_intent) - { - for r in exact { - if !all_exact.iter().any(|e| e.chunk_id == r.chunk_id) { - all_exact.push(r); - } - } - } - } - - tracing::debug!( - "MCP: FTS found {} results, exact found {} results", - fts_results.len(), - all_exact.len() - ); - - rrf_fusion_with_exact( - &vector_results, - &fts_results, - &all_exact, - vector_k as f32, - fts_k as f32, - EXACT_MATCH_RRF_K, - ) - }; - - Ok(fused) - }, - ctx.stores.clone(), - ) - .await - { - Ok(fused) => { - // Map FusedResult back to SearchResult - let chunk_to_result: std::collections::HashMap< - u32, - &crate::vectordb::SearchResult, - > = vector_results.iter().map(|r| (r.id, r)).collect(); - - let mut mapped: Vec = Vec::new(); - for f in fused.into_iter().take(limit) { - if let Some(result) = chunk_to_result.get(&f.chunk_id) { - let mut r = (*result).clone(); - r.score = f.rrf_score; - mapped.push(r); - } - } - mapped - } - Err(e) => { - tracing::warn!("MCP: FTS store unavailable, using vector-only: {:?}", e); - // Degrading to vector-only is correct, but it must be VISIBLE: - // a caller that gets half a hybrid search with no signal cannot - // tell it from a complete one. - single_warnings.push(format!("lexical (FTS) search failed: {e:#}")); - vector_results.into_iter().take(limit).collect() - } - }; - - // Apply language boost - if let Some((_, _, Some(primary_lang))) = crate::search::read_metadata(&self.db_path) { - for result in &mut results { - let file_lang = format!( - "{:?}", - Language::from_path(std::path::Path::new(&result.path)) - ); - if file_lang.to_lowercase() == primary_lang.to_lowercase() { - result.score *= 1.2; - } - } - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - } - - // Apply kind boost - if let Some(target_kind) = structural_intent { - boost_kind(&mut results, target_kind); - } - - // Auto-fallback: if hybrid search returned very few results for a code-like query, - // run literal FTS and merge missing chunks. - if results.len() < 3 && has_identifiers { - tracing::debug!( - "Auto-fallback: semantic returned {} results, trying literal", - results.len() - ); - - let literal_results = self - .with_fts_store_read_for( - |fts_store| fts_store.search(&request.query, limit, None), - ctx.stores.clone(), - ) - .await - .unwrap_or_default(); - - let mut existing_ids: std::collections::HashSet = - results.iter().map(|r| r.id).collect(); - - for fts in literal_results { - if results.len() >= limit { - break; - } - if existing_ids.contains(&fts.chunk_id) { - continue; - } - - let maybe_resolved = self - .with_vector_store_read_for( - |store| { - if let Ok(Some(chunk)) = store.get_chunk(fts.chunk_id) { - Ok(Some(crate::vectordb::SearchResult { - id: fts.chunk_id, - content: chunk.content, - path: chunk.path, - start_line: chunk.start_line, - end_line: chunk.end_line, - kind: chunk.kind, - signature: chunk.signature, - docstring: chunk.docstring, - context: chunk.context, - hash: chunk.hash, - distance: 0.0, - score: fts.score, - context_prev: chunk.context_prev, - context_next: chunk.context_next, - })) - } else { - Ok(None) - } - }, - ctx.stores.clone(), - ) - .await - .ok() - .flatten(); - - if let Some(resolved) = maybe_resolved { - existing_ids.insert(resolved.id); - results.push(resolved); - } - } - } - - tracing::debug!("MCP: Final {} results after hybrid search", results.len()); - self.build_semantic_response( - results, - &request, - compact, - has_identifiers, - ctx.project_alias.as_deref(), - &ctx.alias_roots, - &single_warnings, - ) - } - - // === Helper methods (not exposed as tools) === - - /// Multi-store semantic search: fan out across all stores, merge raw vector/FTS - /// results, then apply RRF fusion. - #[allow(clippy::too_many_arguments)] - async fn semantic_search_multi( - &self, - request: &SemanticSearchRequest, - identifiers: &[String], - limit: usize, - compact: bool, - stores: Vec>, - aliases: &[String], - alias_roots: &std::collections::HashMap, - ) -> Result { - let mode = request.mode.as_deref().unwrap_or("auto"); - let structural_intent = detect_structural_intent(&request.query); - - // === Lexical mode: FTS only across all stores === - if mode == "lexical" { - // Lexical has no second backend, so a failed store here is invisible - // unless it is reported: the query simply looks like it found nothing. - let mut lexical_warnings: Vec = Vec::new(); - - let outcome = self - .with_fts_store_read_multi( - |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), - stores.clone(), - aliases, - ) - .await - .unwrap_or_default(); - if !outcome.failures.is_empty() { - tracing::error!( - "MCP: lexical fan-out degraded — {} of {} repo(s) failed: {:?}", - outcome.failures.len(), - stores.len(), - outcome.failures - ); - lexical_warnings.extend(outcome.warnings("literal search")); - } - let fts_results = outcome.results; - - // Also do exact search if identifiers detected - let mut all_fts = fts_results; - for ident in identifiers { - let exact_outcome = self - .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), - stores.clone(), - aliases, - ) - .await - .unwrap_or_default(); - lexical_warnings.extend(exact_outcome.warnings("exact-identifier search")); - merge_exact_into_fts(&mut all_fts, exact_outcome.results); - } - - all_fts.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let results = self - .resolve_fts_to_search_results_multi( - &all_fts, - limit, - &stores, - aliases, - &mut lexical_warnings, - ) - .await; - - if let Some(target_kind) = structural_intent { - // We need mutable results but we have them as vectordb::SearchResult - let mut mutable_results = results; - boost_kind(&mut mutable_results, target_kind); - return self.build_semantic_response( - mutable_results, - request, - compact, - !identifiers.is_empty(), - None, - alias_roots, - &lexical_warnings, - ); - } - - return self.build_semantic_response( - results, - request, - compact, - !identifiers.is_empty(), - None, - alias_roots, - &lexical_warnings, - ); - } - - // === Modes requiring embedding: "semantic", "hybrid", "auto" === - let query_embedding = { - let mut service_guard = match self.get_embedding_service() { - Ok(g) => g, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error initializing embedding service: {e:#}" - ))])); - } - }; - let service = service_guard.as_mut().unwrap(); - match service.embed_query(&request.query) { - Ok(e) => e, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error embedding query: {e:#}" - ))])); - } - } - }; - - // Search vector stores across all repos - let outcome = self - .with_vector_store_read_multi( - |store| { - store - .search(&query_embedding, limit * 5) - .context("Error searching vector store") - }, - stores.clone(), - aliases, - ) - .await; - - // Warnings raised by the fan-out, carried into the response so the - // calling agent can tell "not in the corpus" from "that repo is down". - let mut search_warnings: Vec = Vec::new(); - - let vector_results = - match outcome { - Ok(o) => { - if !o.failures.is_empty() { - tracing::error!( - "MCP: vector fan-out degraded — {} of {} repo(s) failed: {:?}", - o.failures.len(), - stores.len(), - o.failures - ); - // Only "semantic" has no second backend to fall back on. In - // hybrid/auto/lexical the FTS half can still answer, so - // hard-failing here would throw away good results — the same - // reason one broken repo does not abort the whole fan-out. - if mode == "semantic" && o.results.is_empty() { - let detail = o - .failures - .iter() - .map(|(alias, err)| format!(" - {alias}: {err}")) - .collect::>() - .join("\n"); - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching vector store: {} of {} repo(s) in scope failed \ - and none returned results:\n{}", - o.failures.len(), - stores.len(), - detail - ))])); - } - search_warnings.extend(o.failures.iter().map(|(alias, err)| { - format!("repo '{alias}' vector search failed: {err}") - })); - } - o.results - } - Err(e) => { - tracing::error!("MCP: vector fan-out failed: {:?}", e); - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching vector store: {e:#}" - ))])); - } - }; - - // === Mode: "semantic" — vector only === - if mode == "semantic" { - let fused = vector_only(&vector_results); - let chunk_to_result: std::collections::HashMap = - vector_results.iter().map(|r| (r.id, r)).collect(); - - let mut results: Vec = Vec::new(); - for f in fused.into_iter().take(limit) { - if let Some(result) = chunk_to_result.get(&f.chunk_id) { - let mut r = (*result).clone(); - r.score = f.rrf_score; - results.push(r); - } - } - return self.build_semantic_response( - results, - request, - compact, - !identifiers.is_empty(), - None, - alias_roots, - &search_warnings, - ); - } - - // === Modes: "hybrid" | "auto" — full hybrid search === - let (vector_k, fts_k) = adapt_rrf_k(&request.query); - - // FTS search across all stores. Its failures matter as much as the - // vector half's: during the cloud read-only incident literal search - // also returned 0 results for every affected vendor, and looked clean. - let fts_outcome = self - .with_fts_store_read_multi( - |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), - stores.clone(), - aliases, - ) - .await - .unwrap_or_default(); - if !fts_outcome.failures.is_empty() { - tracing::error!( - "MCP: FTS fan-out degraded — {} of {} repo(s) failed: {:?}", - fts_outcome.failures.len(), - stores.len(), - fts_outcome.failures - ); - search_warnings.extend(fts_outcome.warnings("literal search")); - } - let fts_results = fts_outcome.results; - - // Exact identifier search across all stores - let all_exact = if !identifiers.is_empty() { - let mut exact_results: Vec = Vec::new(); - for ident in identifiers { - let exact_outcome = self - .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), - stores.clone(), - aliases, - ) - .await - .unwrap_or_default(); - search_warnings.extend(exact_outcome.warnings("exact-identifier search")); - for r in exact_outcome.results { - if !exact_results.iter().any(|e| e.chunk_id == r.chunk_id) { - exact_results.push(r); - } - } - } - exact_results - } else { - Vec::new() - }; - - // RRF fusion - let fused = if identifiers.is_empty() { - rrf_fusion(&vector_results, &fts_results, vector_k as f32) - } else { - rrf_fusion_with_exact( - &vector_results, - &fts_results, - &all_exact, - vector_k as f32, - fts_k as f32, - EXACT_MATCH_RRF_K, - ) - }; - - // Map FusedResult back to SearchResult via chunk lookup across all stores - let chunk_to_result: std::collections::HashMap = - vector_results.iter().map(|r| (r.id, r)).collect(); - - let mut mapped: Vec = Vec::new(); - for f in fused.into_iter().take(limit) { - if let Some(result) = chunk_to_result.get(&f.chunk_id) { - let mut r = (*result).clone(); - r.score = f.rrf_score; - mapped.push(r); - } else { - // Chunk from FTS but not in vector results — resolve from stores - if let Some(resolved) = self - .resolve_chunk_from_stores( - f.chunk_id, - f.rrf_score, - &stores, - aliases, - &mut search_warnings, - ) - .await - { - mapped.push(resolved); - } - } - } - - // Apply kind boost - if let Some(target_kind) = structural_intent { - boost_kind(&mut mapped, target_kind); - } - - self.build_semantic_response( - mapped, - request, - compact, - !identifiers.is_empty(), - None, - alias_roots, - &search_warnings, - ) - } - - /// Resolve a single chunk from multiple stores (used for FTS-only hits in multi-store fusion). - async fn resolve_chunk_from_stores( - &self, - chunk_id: u32, - score: f32, - stores: &[Arc], - aliases: &[String], - warnings: &mut Vec, - ) -> Option { - for (idx, store_arc) in stores.iter().enumerate() { - let store = store_arc.vector_store.read().await; - let looked_up = store.get_chunk(chunk_id); - if let Err(ref e) = looked_up { - note_store_failure(warnings, aliases, idx, "chunk lookup", e); - } - if let Ok(Some(chunk)) = looked_up { - return Some(crate::vectordb::SearchResult { - id: chunk_id, - content: chunk.content, - path: chunk.path, - start_line: chunk.start_line, - end_line: chunk.end_line, - kind: chunk.kind, - signature: chunk.signature, - docstring: chunk.docstring, - context: chunk.context, - hash: chunk.hash, - distance: 0.0, - score, - context_prev: chunk.context_prev, - context_next: chunk.context_next, - }); - } - } - None - } - - /// Resolve FTS results to SearchResult using multiple stores. - async fn resolve_fts_to_search_results_multi( - &self, - fts_results: &[crate::fts::FtsResult], - limit: usize, - stores: &[Arc], - aliases: &[String], - warnings: &mut Vec, - ) -> Vec { - let mut results = Vec::new(); - for fts in fts_results.iter().take(limit) { - for (idx, store_arc) in stores.iter().enumerate() { - let store = store_arc.vector_store.read().await; - let looked_up = store.get_chunk(fts.chunk_id); - if let Err(ref e) = looked_up { - // `Ok(None)` means "this store does not hold that chunk" and - // is normal during fan-out; `Err` means the store is broken. - // Collapsing the two is how a dead vector store renders as - // an empty literal search — the exact shape of the step-8 - // incident, which tantivy-side checks cannot detect. - note_store_failure(warnings, aliases, idx, "chunk lookup", e); - } - if let Ok(Some(chunk)) = looked_up { - results.push(crate::vectordb::SearchResult { - id: fts.chunk_id, - content: chunk.content, - path: chunk.path, - start_line: chunk.start_line, - end_line: chunk.end_line, - kind: chunk.kind, - signature: chunk.signature, - docstring: chunk.docstring, - context: chunk.context, - hash: chunk.hash, - distance: 0.0, - score: fts.score, - context_prev: chunk.context_prev, - context_next: chunk.context_next, - }); - break; // Found in this store, skip remaining stores - } - } - } - results - } - - /// Lexical-only search: FTS without embedding service. - #[allow(clippy::too_many_arguments)] - async fn semantic_search_lexical( - &self, - request: &SemanticSearchRequest, - identifiers: &[String], - limit: usize, - compact: bool, - stores: Option>, - project_alias: Option<&str>, - alias_roots: &std::collections::HashMap, - ) -> Result { - let structural_intent = detect_structural_intent(&request.query); - - // `project=`-scoped queries route here, not through the fan-out - // (`is_multi` requires >1 store), so this path needs the same failure - // reporting — it is at least as common as a group query. - let mut lexical_warnings: Vec = Vec::new(); - - let mut fts_results = match self - .with_fts_store_read_for( - |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), - stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - let msg = format!("literal search failed: {e:#}"); - tracing::error!("MCP: {}", msg); - lexical_warnings.push(msg); - Vec::new() - } - }; - - // Also do exact search if identifiers detected - for ident in identifiers { - let exact = match self - .with_fts_store_read_for( - |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), - stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - let msg = format!("exact-identifier search for '{ident}' failed: {e:#}"); - tracing::error!("MCP: {}", msg); - lexical_warnings.push(msg); - continue; - } - }; - merge_exact_into_fts(&mut fts_results, exact); - } - - fts_results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - // Resolve FTS results to chunk metadata - let mut results = self - .resolve_fts_to_search_results(&fts_results, limit, stores, &mut lexical_warnings) - .await; - - // Apply kind boost - if let Some(target_kind) = structural_intent { - boost_kind(&mut results, target_kind); - } - - self.build_semantic_response( - results, - request, - compact, - !identifiers.is_empty(), - project_alias, - alias_roots, - &lexical_warnings, - ) - } - - /// Build the final SemanticSearchResponse with low-confidence signaling. - // Eight parameters, one over clippy's threshold. Bundling them into a - // `ResponseContext` struct is the right end state and is recorded as a - // follow-up; doing it in an incident fix would touch all seven call sites - // for no behavioural gain. The alternative — dropping `warnings` — is not - // acceptable: without it a failed repo is silently reported as "no match". - #[allow(clippy::too_many_arguments)] - fn build_semantic_response( - &self, - results: Vec, - request: &SemanticSearchRequest, - compact: bool, - has_identifiers: bool, - project_alias: Option<&str>, - alias_roots: &std::collections::HashMap, - // Repos that failed during a fan-out. MUST reach the caller: the - // consumer of this tool is a remote agent that never sees the server - // log, so a silently omitted repo reads as "no match there" — a false - // negative. The federated path already does this (`warnings` on the - // remote-project fan-out); the local path never could. - warnings: &[String], - ) -> Result { - let warnings = if warnings.is_empty() { - None - } else { - Some(warnings.to_vec()) - }; - if results.is_empty() { - let response = SemanticSearchResponse { - results: vec![], - low_confidence: Some(true), - suggested_tool: retry_hint(Some("literal_search".to_string()), &warnings), - warnings, - }; - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - return Ok(CallToolResult::success(vec![Content::text(json)])); - } - - // Pre-compute normalized project root for stripping absolute paths - let project_root_normalized = { - let root = crate::cache::normalize_path_str(self.project_path.to_str().unwrap_or("")); - root.trim_end_matches('/').to_string() - }; - - let mut items: Vec = results - .into_iter() - .filter(|r| { - if let Some(ref fp) = request.filter_path { - let normalized_filter = crate::cache::normalize_filter_path(fp); - if normalized_filter.is_empty() { - return true; - } - // Relativise against the ROUTED project's root, not the - // service's own project_path — otherwise a serve-routed - // absolute path never strips and every hit is dropped. - let filter_root = pick_filter_root( - &r.path, - project_alias, - alias_roots, - &project_root_normalized, - ); - crate::cache::path_matches_filter(&r.path, &normalized_filter, &filter_root) - } else { - true - } - }) - .map(|r| SearchResultItem { - chunk_id: r.id, - path: r.path, - start_line: r.start_line, - end_line: r.end_line, - kind: r.kind, - score: r.score, - signature: r.signature, - content: if compact { None } else { Some(r.content) }, - context_prev: if compact { None } else { r.context_prev }, - context_next: if compact { None } else { r.context_next }, - source: None, - chunk_ref: None, - }) - .collect(); - - // Prefix paths with alias for multi-repo / single-project identification - for item in &mut items { - if let Some(alias) = project_alias { - if let Some(root) = alias_roots.get(alias) { - item.path = prefix_path_with_alias(&item.path, Some(alias), root); - } else { - item.path = crate::cache::normalize_path_str(&item.path); - } - } else if !alias_roots.is_empty() { - item.path = prefix_path_multi(&item.path, &[], alias_roots); - } - } - - // Check low-confidence: top result's RRF score below threshold - let top_score = items.first().map(|r| r.score); - let (low_confidence, suggested_tool) = compute_low_confidence(top_score, has_identifiers); - let suggested_tool = retry_hint(suggested_tool, &warnings); - - let response = SemanticSearchResponse { - results: items, - low_confidence, - suggested_tool, - warnings, - }; - - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) - } - - /// Resolve FTS results to SearchResult by looking up chunk metadata. - async fn resolve_fts_to_search_results( - &self, - fts_results: &[crate::fts::FtsResult], - limit: usize, - stores: Option>, - warnings: &mut Vec, - ) -> Vec { - let outcome = self - .with_vector_store_read_for( - |store| { - let mut results = Vec::new(); - for fts in fts_results.iter().take(limit) { - // A failed lookup is not an absent chunk. Propagating the - // error keeps a broken vector store from rendering as an - // ordinary empty literal search. - let chunk = store - .get_chunk(fts.chunk_id) - .context("Error resolving FTS hit to chunk metadata")?; - if let Some(chunk) = chunk { - results.push(crate::vectordb::SearchResult { - id: fts.chunk_id, - content: chunk.content, - path: chunk.path, - start_line: chunk.start_line, - end_line: chunk.end_line, - kind: chunk.kind, - signature: chunk.signature, - docstring: chunk.docstring, - context: chunk.context, - hash: chunk.hash, - distance: 0.0, - score: fts.score, - context_prev: chunk.context_prev, - context_next: chunk.context_next, - }); - } - } - Ok(results) - }, - stores, - ) - .await; - match outcome { - Ok(results) => results, - Err(e) => { - let msg = format!("literal search could not read the index: {e:#}"); - tracing::error!("MCP: {}", msg); - if !warnings.contains(&msg) { - warnings.push(msg); - } - Vec::new() - } - } - } - - // === find_definition internal === - - /// Internal: find symbol definitions, used by `find(kind="definition")`. - async fn find_definition( - &self, - Parameters(request): Parameters, - ) -> Result { - let limit = request.limit.unwrap_or(20); - - tracing::debug!( - "MCP find_definition: symbol='{}', kind={:?}, limit={}", - request.symbol, - request.kind, - limit - ); - - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "find") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - // Stores that failed during this lookup. Without this, "the symbol may - // not be indexed" below is emitted as a confident diagnosis even when - // no store ever answered. - let mut find_warnings: Vec = Vec::new(); - - // FTS search — multi-store or single - let fts_results = if let Some(ref sv) = ctx.stores_vec { - let sa = ctx.store_aliases.as_ref().unwrap(); - self.with_fts_store_read_multi( - |fts_store| fts_store.search(&request.symbol, limit * 3, None), - sv.clone(), - sa, - ) - .await - .unwrap_or_default() - .into_results(&mut find_warnings, "definition search") - } else { - match self - .with_fts_store_read_for( - |fts_store| fts_store.search(&request.symbol, limit * 3, None), - ctx.stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {e:#}" - ))])); - } - } - }; - - if fts_results.is_empty() { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result( - format!( - "No definition found for '{}'. The symbol may not be indexed.", - request.symbol - ), - &find_warnings, - ), - )])); - } - - // Resolve chunk metadata and filter by definition kinds - let requested_kind = request.kind.clone(); - let mut items: Vec = if let Some(ref sv) = ctx.stores_vec { - let mut items: Vec = Vec::new(); - 'outer: for fts_result in &fts_results { - for store_arc in sv { - let store = store_arc.vector_store.read().await; - if let Ok(Some(chunk)) = store.get_chunk(fts_result.chunk_id) { - // Skip non-definition kinds — try next FTS result, not next store - if !DEFINITION_KINDS.contains(&chunk.kind.as_str()) { - continue 'outer; - } - if let Some(ref rk) = requested_kind { - if chunk.kind != *rk { - continue 'outer; - } - } - items.push(ReferenceItem { - chunk_id: fts_result.chunk_id, - path: chunk.path, - line: chunk.start_line, - kind: chunk.kind, - signature: chunk.signature, - score: fts_result.score, - }); - if items.len() >= limit { - break 'outer; - } - break; // Found in this store — move to next FTS result - } - } - // If we get here, chunk wasn't found in any store — just skip it - } - items - } else { - match self - .with_vector_store_read_for( - |store| { - let items = fts_results - .iter() - .filter_map(|fts_result| { - if let Ok(Some(chunk)) = store.get_chunk(fts_result.chunk_id) { - if !DEFINITION_KINDS.contains(&chunk.kind.as_str()) { - return None; - } - if let Some(ref requested_kind) = requested_kind { - if chunk.kind != *requested_kind { - return None; - } - } - Some(ReferenceItem { - chunk_id: fts_result.chunk_id, - path: chunk.path, - line: chunk.start_line, - kind: chunk.kind, - signature: chunk.signature, - score: fts_result.score, - }) - } else { - None - } - }) - .take(limit) - .collect(); - Ok(items) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error opening database: {e:#}" - ))])); - } - } - }; - - // Prefix paths with alias for multi-repo identification - for item in &mut items { - item.path = ctx.prefix_result_path(&item.path); - } - - respond_with_items(&items, &find_warnings, || { - format!( - "No definition found for '{}'. Try find_usages() to find references, \ - or broaden your search.", - request.symbol - ) - }) - } - - // === find_usages tool === - - async fn find_usages( - &self, - Parameters(request): Parameters, - ) -> Result { - self.find_usages_impl( - request.symbol.clone(), - request.limit.unwrap_or(20), - request.project, - request.group, - ) - .await - } - - /// Shared implementation for find_usages (used by `find(kind="usages")`). - async fn find_usages_impl( - &self, - symbol: String, - limit: usize, - project: Option, - group: Option, - ) -> Result { - tracing::debug!("MCP find_usages: symbol='{}', limit={}", symbol, limit); - - // Resolve project/group routing - let ctx = match self.resolve_routing(&project, &group, false, "find").await { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - // See `find_definition`: an empty result and a dead store must not - // produce the same sentence. - let mut find_warnings: Vec = Vec::new(); - - // FTS search — multi-store or single - let fts_results = if let Some(ref sv) = ctx.stores_vec { - let sa = ctx.store_aliases.as_ref().unwrap(); - self.with_fts_store_read_multi( - |fts_store| fts_store.search(&symbol, limit * 2, None), - sv.clone(), - sa, - ) - .await - .unwrap_or_default() - .into_results(&mut find_warnings, "usage search") - } else { - match self - .with_fts_store_read_for( - |fts_store| fts_store.search(&symbol, limit * 2, None), - ctx.stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {e:#}" - ))])); - } - } - }; - - if fts_results.is_empty() { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result( - format!("No usages found for '{symbol}'. The symbol may not be indexed."), - &find_warnings, - ), - )])); - } - - // Resolve chunks and exclude definition chunks - let mut items: Vec = if let Some(ref sv) = ctx.stores_vec { - let mut items: Vec = Vec::new(); - for fts_result in &fts_results { - for store_arc in sv { - let store = store_arc.vector_store.read().await; - if let Ok(Some(chunk)) = store.get_chunk(fts_result.chunk_id) { - if !is_definition_chunk(&chunk.kind, &chunk.signature, &symbol) { - items.push(ReferenceItem { - chunk_id: fts_result.chunk_id, - path: chunk.path, - line: chunk.start_line, - kind: chunk.kind, - signature: chunk.signature, - score: fts_result.score, - }); - } - break; - } - } - if items.len() >= limit { - break; - } - } - items - } else { - match self - .with_vector_store_read_for( - |store| { - let items = fts_results - .iter() - .filter_map(|fts_result| { - if let Ok(Some(chunk)) = store.get_chunk(fts_result.chunk_id) { - if is_definition_chunk(&chunk.kind, &chunk.signature, &symbol) { - return None; - } - Some(ReferenceItem { - chunk_id: fts_result.chunk_id, - path: chunk.path, - line: chunk.start_line, - kind: chunk.kind, - signature: chunk.signature, - score: fts_result.score, - }) - } else { - None - } - }) - .take(limit) - .collect(); - Ok(items) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error opening database: {e:#}" - ))])); - } - } - }; - - // Prefix paths with alias for multi-repo identification - for item in &mut items { - item.path = ctx.prefix_result_path(&item.path); - } - - respond_with_items(&items, &find_warnings, || { - format!( - "No usages found for '{symbol}' (only definitions were found). Try \ - find_definition() to locate the declaration." - ) - }) - } - - /// Fetch outline items for an already-normalised absolute path. - /// - /// Returns `Ok(vec![])` when no chunks match. - /// In multi-store mode, per-store I/O failures are recorded in `warnings` and - /// skipped (never `Err`) so one broken repo cannot blank the whole outline. - /// In single-store mode, I/O failures are returned as `Err`. - /// - /// `warnings` is not optional: without it a failed store is indistinguishable - /// from a file with no indexed chunks, and the caller is told the file is not - /// indexed — a diagnosis, and a wrong one. - async fn outline_items_for_normalized( - &self, - normalized: &str, - ctx: &MultiStoreContext, - warnings: &mut Vec, - ) -> anyhow::Result> { - if let Some(ref sv) = ctx.stores_vec { - let aliases = ctx.aliases(); - let mut all_items: Vec = Vec::new(); - let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); - for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.chunks_for_file(normalized) { - Ok(metas) => { - for c in metas { - if seen_ids.insert(c.id) { - all_items.push(FileOutlineItem { - chunk_id: c.id, - kind: c.kind, - signature: c.signature, - start_line: c.start_line, - end_line: c.end_line, - }); - } - } - } - Err(ref e) => { - note_store_failure(warnings, aliases, store_idx, "outline scan", e); - } - } - } - all_items.sort_by_key(|i| i.start_line); - Ok(all_items) - } else { - let normalized_owned = normalized.to_string(); - self.with_vector_store_read_for( - move |store| { - let mut out: Vec = store - .chunks_for_file(&normalized_owned)? - .into_iter() - .map(|c| FileOutlineItem { - chunk_id: c.id, - kind: c.kind, - signature: c.signature, - start_line: c.start_line, - end_line: c.end_line, - }) - .collect(); - out.sort_by_key(|i| i.start_line); - Ok(out) - }, - ctx.stores.clone(), - ) - .await - } - } - - async fn file_outline( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "explore") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - // Outline operates on a single repo — reject group fan-out - if ctx.is_multi { - return Ok(CallToolResult::success(vec![Content::text( - "Tool 'explore' operates on a single repo. Use 'project' instead of 'group'." - .to_string(), - )])); - } - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - // In serve mode, use the resolved project root from alias_roots; - // self.project_path is "serve://multi-repo" which doesn't resolve. - let project_root = if let Some(ref alias) = ctx.project_alias { - ctx.alias_roots - .get(alias) - .map(PathBuf::from) - .unwrap_or_else(|| self.project_path.clone()) - } else { - self.project_path.clone() - }; - // Strip project-alias prefix from target path if present. - // E.g. "ExampleRepo/src/foo.cs" with project="ExampleRepo" → "src/foo.cs" - let stripped_path = strip_alias_prefix(&request.path, ctx.project_alias.as_ref()); - let normalized = normalize_tool_path(&stripped_path, &project_root); - - let mut outline_warnings: Vec = Vec::new(); - let mut items = match self - .outline_items_for_normalized(&normalized, &ctx, &mut outline_warnings) - .await - { - Ok(v) => v, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error reading outline: {e:#}" - ))])); - } - }; - - // Two-pass fallback: if alias-stripping changed the path and yielded no results, - // try the original un-stripped path. Handles the case where the project alias - // matches a package subdirectory name (e.g. project "my_pkg" with target - // "my_pkg/config.py" → after strip becomes "config.py" which is wrong; - // the correct relative path is "my_pkg/config.py"). - if items.is_empty() && stripped_path != request.path { - let normalized_orig = normalize_tool_path(&request.path, &project_root); - if normalized_orig != normalized { - tracing::debug!( - "file_outline: primary '{}' empty, trying fallback '{}'", - normalized, - normalized_orig - ); - items = match self - .outline_items_for_normalized(&normalized_orig, &ctx, &mut outline_warnings) - .await - { - Ok(v) => v, - Err(e) => { - tracing::warn!( - "file_outline: fallback '{}' also failed: {:?}", - normalized_orig, - e - ); - push_store_warning( - &mut outline_warnings, - &store_warning( - ctx.project_alias.as_deref().unwrap_or("unknown"), - "outline scan", - &format!("{e:#}"), - ), - ); - Vec::new() - } - }; - } - } - - respond_with_items(&items, &outline_warnings, || { - "No indexed chunks found for path. Verify the file is within the \ - project root and the index is up to date." - .to_string() - }) - } - - #[tool( - description = "Retrieve the full content of a specific chunk by its ID, plus optional surrounding lines for context.\nUse this after search or explore to read the actual code without loading the whole file.\n\nUSE FOR: reading a specific function/class body after finding it via search.\nSet context_lines (default 0, max 20) to include lines before and after the chunk.\n\nIMPORTANT (multi-repo): chunk_ids are local to each repository and are NOT globally unique.\nWhen `project` is omitted in multi-repo mode, the tool scans all repositories for the chunk_id.\nIf found in exactly one repo, it is returned automatically. If found in multiple repos, an `ambiguous_chunk_id` error lists the candidates so you can retry with `project`." - )] - async fn get_chunk( - &self, - Parameters(request): Parameters, - ) -> Result { - tracing::info!( - "📥 get_chunk(chunk_id={}, project={:?})", - request.chunk_id, - request.project, - ); - - // Federation: a `chunk_ref` of the form "/:" - // (returned by a federated search result) fetches the chunk from a remote - // peer rather than the local index. The alias scopes the fetch to a single - // remote project so the multi-repo peer can disambiguate the chunk_id. - if let Some(chunk_ref) = request.chunk_ref.as_deref() { - return self - .federated_get_chunk(chunk_ref, request.context_lines) - .await; - } - - // In multi-repo serve mode, require explicit project or group scope. - // Unscoped get_chunk would fan-out over all repos, opening all DBs unnecessarily. - // Consistent with search/find/explore which also require scope. - if request.project.is_none() && request.group.is_none() { - if let Some(ref serve_state) = self.serve_state { - let config = serve_state.config_snapshot(); - if config.repos.len() > 1 { - return Ok(CallToolResult::success(vec![Content::text( - self.format_scope_error(), - )])); - } - } - } - - // Resolve project/group routing — allow unscoped only for single-repo mode - let ctx = match self - .resolve_routing(&request.project, &request.group, true, "get_chunk") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - let mut clamped = false; - let mut context_lines = request.context_lines.unwrap_or(0); - if context_lines > 20 { - context_lines = 20; - clamped = true; - } - - // Stores that failed while looking up this chunk. get_chunk previously - // collapsed every `Err` into "not found", so during the read-only - // incident it would have reported every chunk in every vendor repo as - // missing — a confident, wrong answer. - let mut chunk_warnings: Vec = Vec::new(); - - // Look up chunk — multi-store: smart candidate detection for chunk_id collision. - // chunk_ids are local per database, not globally unique. When no project is specified - // and multiple stores are active, scan all stores to find which ones have this chunk_id. - let chunk = if let Some(ref sv) = ctx.stores_vec { - if sv.len() > 1 && request.project.is_none() { - // Smart candidate detection: find which stores actually contain this chunk_id - let mut candidates: Vec<(&Arc, String)> = Vec::new(); - let aliases = ctx.aliases(); - for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.get_chunk(request.chunk_id) { - Ok(Some(_)) => { - // A store that HAS the chunk stays a candidate even if - // its alias is missing. `resolve_repo_stores_multi` - // keeps stores and aliases the same length, so this is - // unreachable today — but gating the push on - // `aliases.get(i)` meant a future break of that - // invariant would degrade to a silent auto-route rather - // than a loud one. The placeholder is per-index so two - // aliasless candidates stay distinguishable in - // `candidate_projects`. - let alias = aliases - .get(i) - .cloned() - .unwrap_or_else(|| format!("")); - candidates.push((store_arc, alias)); - } - Ok(None) => continue, - Err(ref e) => { - note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", e); - continue; - } - } - } - match candidates.len() { - 0 => { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result( - format!( - "Chunk {} not found in any repository. Verify the \ - chunk_id and index state.", - request.chunk_id - ), - &chunk_warnings, - ), - )])); - } - 1 => { - // Exactly one store has this chunk_id — auto-route - let (store_arc, ref alias) = candidates[0]; - // Record tool call for the specific repo that served this chunk - if let Some(ref serve_state) = self.serve_state { - serve_state.record_tool_call(alias, "get_chunk"); - serve_state.touch_access(alias); - } - let store = store_arc.vector_store.read().await; - match store.get_chunk(request.chunk_id) { - Ok(c) => c, - Err(ref e) => { - push_store_warning( - &mut chunk_warnings, - &store_warning(alias, "chunk lookup", &format!("{e:#}")), - ); - None - } - } - } - _ => { - // Multiple stores have this chunk_id — ambiguous. - // - // `candidate_projects` reads as the complete list, so a - // store that failed to answer has to be declared: the - // right repo may be the one missing from it. - let candidate_names: Vec<&str> = - candidates.iter().map(|(_, a)| a.as_str()).collect(); - let payload = ambiguous_chunk_payload( - request.chunk_id, - &candidate_names, - &chunk_warnings, - ); - return Ok(CallToolResult::success(vec![Content::text( - payload.to_string(), - )])); - } - } - } else { - // Single store or project specified — direct lookup - let aliases = ctx.aliases(); - let mut found = None; - for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.get_chunk(request.chunk_id) { - Ok(Some(c)) => { - found = Some(c); - break; - } - Ok(None) => continue, - // Do NOT abandon the remaining stores: one broken store - // says nothing about the others, and the chunk may well - // live in a healthy one. - Err(ref e) => { - note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", e); - continue; - } - } - } - found - } - } else { - match self - .with_vector_store_read_for( - |store| store.get_chunk(request.chunk_id), - ctx.stores.clone(), - ) - .await - { - Ok(c) => c, - Err(e) => { - push_store_warning( - &mut chunk_warnings, - &store_warning( - ctx.project_alias.as_deref().unwrap_or("unknown"), - "chunk lookup", - &format!("{e:#}"), - ), - ); - None - } - } - }; - - let mut chunk = match chunk { - Some(c) => c, - None => { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result( - format!( - "Chunk {} not found. Verify the chunk_id and index state.", - request.chunk_id - ), - &chunk_warnings, - ), - )])); - } - }; - - // Prefix path with alias for multi-repo identification - chunk.path = ctx.prefix_result_path(&chunk.path); - - let mut context_before = None; - let mut context_after = None; - let mut note = None; - - if context_lines > 0 { - // Resolve relative chunk paths against project root (not process CWD). - let source_path = if Path::new(&chunk.path).is_absolute() { - PathBuf::from(&chunk.path) - } else { - self.project_path.join(&chunk.path) - }; - match tokio::fs::read_to_string(&source_path).await { - Ok(src) => { - let lines: Vec<&str> = src.lines().collect(); - if !lines.is_empty() { - let before_start = chunk.start_line.saturating_sub(context_lines); - let before_end = chunk.start_line.min(lines.len()); - if before_start < before_end { - context_before = Some(lines[before_start..before_end].join("\n")); - } - - let after_start = chunk.end_line.min(lines.len()); - let after_end = (chunk.end_line + context_lines).min(lines.len()); - if after_start < after_end { - context_after = Some(lines[after_start..after_end].join("\n")); - } - } - } - Err(_) => { - note = Some( - "source file not readable, returning indexed content only".to_string(), - ); - } - } - } - - let response = GetChunkResponse { - chunk_id: request.chunk_id, - path: chunk.path, - start_line: chunk.start_line, - end_line: chunk.end_line, - kind: chunk.kind, - signature: chunk.signature, - content: chunk.content, - context_before, - context_after, - context_lines_clamped: if clamped { Some(true) } else { None }, - note, - }; - - // The success path is the one that used to drop this, and it is the - // dangerous one: a confidently-returned chunk from a group where a store - // failed to answer looks exactly like a chunk from a healthy group. Same - // false negative as an empty result, harder to notice. - respond_with_object(&response, &chunk_warnings) - } - - /// Symbol impact analysis — returns transitive call-sites of a symbol with file/line precision. - /// - /// The recommended tool for "who calls X?" / "what breaks if I rename X?". Uses - /// language-specific semantic analysis (SCIP) to find all references, enabling agents - /// to plan refactors with IDE-class accuracy instead of text-matching grep heuristics. - /// Precision backends ship per language: C# (bundled `scip-csharp` helper, - /// `-with-csharp` releases) and TypeScript (`scip-typescript`, resolved via `npx` - /// or `CODESEARCH_SCIP_TYPESCRIPT`). If no backend is installed for the target - /// language, the response reports it — fall back to `find` with `kind="usages"` - /// (lexical) only then. - #[tool( - description = "Symbol impact analysis — find all references to a symbol with IDE-class precision (SCIP).\n\nThe right tool for \"who calls X?\" / \"what breaks if I rename X?\". Returns transitive call-sites with file/line precision, enabling agents to plan refactors without missing a caller. More accurate than text-based `find kind=\"usages\"` because it understands language semantics.\n\nInput variants:\n- By name: `{ \"symbol_name\": \"FieldDefinition.Validate\", \"project\": \"myrepo\" }`\n- By position: `{ \"file\": \"src/Validation/FieldDefinition.cs\", \"line\": 42, \"project\": \"myrepo\" }`\n\nPrecision backends (SCIP) ship per language; C# (bundled `scip-csharp` helper, `-with-csharp` releases) and TypeScript (via `npx` or `CODESEARCH_SCIP_TYPESCRIPT`) are available today. For Rust/Python/Go/etc., use `find` with `kind=\"usages\"` as a text-based fallback until SCIP backends for those languages ship.\n\nIMPORTANT (multi-repo): always specify `project` (single repo). Omitting `project` in multi-repo mode returns a `scope_required` error." - )] - async fn find_impact( - &self, - Parameters(request): Parameters, - ) -> Result { - tracing::info!( - "📥 find_impact(symbol_name={:?}, file={:?}, line={:?}, language={:?}, project={:?})", - request.symbol_name, - request.file, - request.line, - request.language, - request.project, - ); - - // Validate input: must provide either symbol_name or file+line - let has_name = request - .symbol_name - .as_ref() - .is_some_and(|s| !s.trim().is_empty()); - let has_position = request.file.is_some() && request.line.is_some(); - if !has_name && !has_position { - return Ok(CallToolResult::success(vec![Content::text( - "Must provide either `symbol_name` or both `file` and `line` for position-based lookup.".to_string(), - )])); - } - - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "find_impact") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - // Determine project root and db_path for the symbol index - let (project_root, db_path) = if let Some(ref alias) = ctx.project_alias { - let root = ctx - .alias_roots - .get(alias) - .map(PathBuf::from) - .unwrap_or_else(|| self.project_path.clone()); - // The symbol index DB lives alongside the vector DB - let db = root.join(crate::constants::DB_DIR_NAME); - (root, db) - } else { - // Single-repo / stdio mode: use the service's own paths - (self.project_path.clone(), self.db_path.clone()) - }; - - // Use the shared symbol indexer registry - let registry = &self.symbol_registry; - - // Determine which language to use - let language = request.language.clone().or_else(|| { - // Auto-detect from file extension - request.file.as_ref().and_then(|f| { - let ext = Path::new(f).extension()?.to_str()?.to_lowercase(); - match ext.as_str() { - "cs" => Some(crate::constants::LANG_CSHARP.to_string()), - "ts" | "tsx" | "mts" | "cts" => { - Some(crate::constants::LANG_TYPESCRIPT.to_string()) - } - _ => None, - } - }) - }); - - let indexer: &dyn crate::symbols::SymbolIndexer = match language { - Some(ref lang) => match registry.get(lang) { - Some(i) => i, - None => { - let available = registry.available_languages(); - return Ok(CallToolResult::success(vec![Content::text(format!( - "No symbol indexer for language '{}'. Available languages: {:?}", - lang, available - ))])); - } - }, - None => { - // No language specified and couldn't auto-detect — try all installed - let installed = registry.installed_languages(); - if installed.is_empty() { - return Ok(CallToolResult::success(vec![Content::text( - "No symbol indexers installed. Install the `scip-csharp` helper for C# support, or `scip-typescript` (via npx) for TypeScript support.".to_string(), - )])); - } - // Use the first installed language (MVP: C# or TypeScript) - match registry.get(&installed[0]) { - Some(i) => i, - None => { - unreachable!("installed_languages() returned a language with no indexer") - } - } - } - }; - - // Check if the helper is available - if !indexer.is_available() { - let error = crate::symbols::SymbolIndexError { - error: format!( - "Symbol indexer for '{}' is not available. The helper binary is not installed.", - indexer.language() - ), - available_languages: registry.available_languages(), - hint_for_agent: format!( - "Install the `-with-csharp` release variant, or set {} to the helper path.", - crate::constants::SCIP_CSHARP_HELPER_ENV - ), - }; - return Ok(CallToolResult::success(vec![Content::text( - serde_json::to_string(&error).unwrap_or_else(|_| error.error.clone()), - )])); - } - - // Perform the lookup. - // - // `find_references` may invoke `scip-csharp find-refs` on a cache miss - // (lazy Opt-2 reference resolution). That subprocess can take several minutes - // on a large solution, so we use `block_in_place` to avoid blocking the async - // executor thread. `block_in_place` is safe here: we are inside a - // multi-threaded tokio runtime and do not hold any async locks. - let file_for_pos = if !has_name { - Some(self.normalize_symbol_query_path( - &project_root, - Path::new(request.file.as_ref().unwrap()), - )) - } else { - None - }; - let symbol_name_for_lookup = request.symbol_name.clone(); - let line_for_lookup = request.line; - let result = tokio::task::block_in_place(|| { - if has_name { - indexer.find_references(&db_path, symbol_name_for_lookup.as_ref().unwrap()) - } else { - indexer.find_references_by_position( - &db_path, - &file_for_pos.unwrap(), - line_for_lookup.unwrap(), - ) - } - }); - - match result { - Ok(references) => { - let age = indexer.index_age(&db_path); - let impact = crate::symbols::FindImpactResult { - symbol: request.symbol_name.clone().unwrap_or_else(|| { - format!( - "{}:{}", - request.file.as_deref().unwrap_or("?"), - request.line.unwrap_or(0) - ) - }), - references, - index_age_seconds: age, - language: indexer.language().to_string(), - scope: ctx - .project_alias - .map(|a| format!("project:{}", a)) - .unwrap_or_else(|| "local".to_string()), - }; - let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) - } - Err(e) => Ok(CallToolResult::success(vec![Content::text(format!( - "Symbol lookup failed: {e:#}" - ))])), - } - } - - fn normalize_symbol_query_path(&self, project_root: &Path, file: &Path) -> PathBuf { - if file.is_absolute() { - if let Ok(relative) = file.strip_prefix(project_root) { - return PathBuf::from(relative.to_string_lossy().replace('\\', "/")); - } - } - - PathBuf::from(file.to_string_lossy().replace('\\', "/")) - } - - async fn find_imports( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "find") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - // In serve mode, use the resolved project root from alias_roots - let project_root = if let Some(ref alias) = ctx.project_alias { - ctx.alias_roots - .get(alias) - .map(PathBuf::from) - .unwrap_or_else(|| self.project_path.clone()) - } else { - self.project_path.clone() - }; - // Strip project-alias prefix from target path if present. - let stripped_path = strip_alias_prefix(&request.path, ctx.project_alias.as_ref()); - let normalized = normalize_tool_path(&stripped_path, &project_root); - - // Stores that failed during this lookup, so "no imports found" is never - // reported as fact when a store never answered. - let mut import_warnings: Vec = Vec::new(); - - let mut items = if let Some(ref sv) = ctx.stores_vec { - // Multi-store group fan-out: collect import items from all stores - let import_aliases = ctx.aliases(); - let mut all_items: Vec = Vec::new(); - let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); - for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.chunks_for_file(&normalized) { - Ok(metas) => { - for meta in metas { - if !is_import_kind(&meta.kind) { - continue; - } - if seen_ids.insert(meta.id) { - match store.get_chunk(meta.id) { - Ok(Some(chunk)) => all_items.extend(parse_import_lines( - &chunk.content, - chunk.start_line, - )), - Ok(None) => {} - Err(ref e) => note_store_failure( - &mut import_warnings, - import_aliases, - store_idx, - "chunk lookup", - e, - ), - } - } - } - } - Err(ref e) => { - note_store_failure( - &mut import_warnings, - import_aliases, - store_idx, - "imports scan", - e, - ); - } - } - } - all_items - } else { - match self - .with_vector_store_read_for( - |store| { - let mut out = Vec::new(); - for meta in store.chunks_for_file(&normalized)? { - if !is_import_kind(&meta.kind) { - continue; - } - if let Some(chunk) = store.get_chunk(meta.id)? { - out.extend(parse_import_lines(&chunk.content, chunk.start_line)); - } - } - Ok(out) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error reading imports: {e:#}" - ))])); - } - } - }; - - if items.is_empty() { - // Fallback: no import-kind chunks found for this file. Broaden the - // search to common import keywords and filter to the target path. - // Limitation: this only finds chunks containing these literal words; - // language-specific import forms that lack these keywords will be missed. - let fallback_limit = 40usize; - let mut all_hits: Vec<(u32, f32)> = Vec::new(); - let mut seen_fts_ids: HashSet = HashSet::new(); - - if let Some(ref sv) = ctx.stores_vec { - let import_aliases = ctx.aliases(); - // Multi-store FTS fallback - for keyword in IMPORT_FTS_KEYWORDS { - let hits = self - .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(keyword, fallback_limit, None), - sv.clone(), - ctx.store_aliases.as_ref().unwrap(), - ) - .await - .unwrap_or_default() - .into_results(&mut import_warnings, "imports search"); - for h in hits { - if seen_fts_ids.insert(h.chunk_id) { - all_hits.push((h.chunk_id, h.score)); - } - } - } - - // Resolve FTS hits via vector stores - let mut resolved: Vec = Vec::new(); - for (chunk_id, _) in &all_hits { - for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.get_chunk(*chunk_id) { - Ok(Some(chunk)) => { - if crate::cache::normalize_path_str(&chunk.path) == normalized { - resolved.extend(parse_import_lines( - &chunk.content, - chunk.start_line, - )); - } - break; - } - Ok(None) => continue, - Err(ref e) => { - note_store_failure( - &mut import_warnings, - import_aliases, - store_idx, - "chunk lookup", - e, - ); - continue; - } - } - } - } - items = resolved; - } else { - // Single-store FTS fallback - for keyword in IMPORT_FTS_KEYWORDS { - let hits = match self - .with_fts_store_read_for( - |fts_store| fts_store.search_exact(keyword, fallback_limit, None), - ctx.stores.clone(), - ) - .await - { - Ok(h) => h, - Err(e) => { - push_store_warning( - &mut import_warnings, - &store_warning( - ctx.project_alias.as_deref().unwrap_or("unknown"), - "imports search", - &format!("{e:#}"), - ), - ); - Vec::new() - } - }; - for h in hits { - if seen_fts_ids.insert(h.chunk_id) { - all_hits.push((h.chunk_id, h.score)); - } - } - } - - items = self - .with_vector_store_read_for( - |store| { - let mut out = Vec::new(); - for (chunk_id, _) in &all_hits { - if let Some(chunk) = store.get_chunk(*chunk_id)? { - if crate::cache::normalize_path_str(&chunk.path) == normalized { - out.extend(parse_import_lines( - &chunk.content, - chunk.start_line, - )); - } - } - } - Ok(out) - }, - ctx.stores.clone(), - ) - .await - .unwrap_or_else(|e| { - push_store_warning( - &mut import_warnings, - &store_warning( - ctx.project_alias.as_deref().unwrap_or("unknown"), - "chunk lookup", - &format!("{e:#}"), - ), - ); - Vec::new() - }); - } - } - - items.sort_by_key(|i| i.line); - respond_with_items(&items, &import_warnings, || { - "No import chunks found. The index may not include import statements \ - for this language, or the file has no imports." - .to_string() - }) - } - - async fn find_dependents( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "find") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - let limit = request.limit.unwrap_or(20).min(200); - let high_limit = (limit * 10).max(200); // generous budget for filtering - - // Stores that failed during this lookup, so "no dependents" is never - // reported as fact when a store never answered. - let mut dep_warnings: Vec = Vec::new(); - - // Extract a meaningful search term from path-like inputs. - // Import chunks contain module references like `use crate::constants::X` - // but the tool receives file paths like `src/constants.rs`. - // We extract the file stem to match against module names in imports. - let search_term = if request.symbol_or_path.contains('/') - || request.symbol_or_path.contains('\\') - || request.symbol_or_path.contains('.') - { - std::path::Path::new(&request.symbol_or_path) - .file_stem() - .and_then(|s| s.to_str()) - .filter(|s| !s.is_empty()) - .unwrap_or(&request.symbol_or_path) - .to_string() - } else { - request.symbol_or_path.clone() - }; - - let import_kind = Some(crate::chunker::ChunkKind::Imports); - - // Two-phase search strategy: - // 1. `search_exact` — precise term match on signature+content with - // MUST filter for Import kind. Strictly limits results to import chunks. - // 2. If that yields no import-kind results, fall back to `search` - // (QueryParser, broader tokenization) with kind boost for imports. - // - // Limitation: the chunker does not emit per-statement AST import chunks; - // imports are gap-classified as `Imports` kind. Chunks whose kind doesn't - // match `is_import_kind()` will be missed regardless of search method. - let fts_results = if let Some(ref sv) = ctx.stores_vec { - let sa = ctx.store_aliases.as_ref().unwrap(); - // Multi-store FTS search - let exact_hits = self - .with_fts_store_read_multi( - |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), - sv.clone(), - sa, - ) - .await - .unwrap_or_default() - .into_results(&mut dep_warnings, "dependents search"); - - if exact_hits.is_empty() { - self.with_fts_store_read_multi( - |fts_store| fts_store.search(&search_term, high_limit, import_kind), - sv.clone(), - sa, - ) - .await - .unwrap_or_default() - .into_results(&mut dep_warnings, "dependents search") - } else { - exact_hits - } - } else { - // Single-store FTS search - let alias = ctx.project_alias.as_deref().unwrap_or("unknown"); - let mut run = |r: anyhow::Result>| match r { - Ok(hits) => hits, - Err(e) => { - push_store_warning( - &mut dep_warnings, - &store_warning(alias, "dependents search", &format!("{e:#}")), - ); - Vec::new() - } - }; - let exact_hits = run(self - .with_fts_store_read_for( - |fts_store| fts_store.search_exact(&search_term, high_limit, import_kind), - ctx.stores.clone(), - ) - .await); - - if exact_hits.is_empty() { - run(self - .with_fts_store_read_for( - |fts_store| fts_store.search(&search_term, high_limit, import_kind), - ctx.stores.clone(), - ) - .await) - } else { - exact_hits - } - }; - - let mut items = if let Some(ref sv) = ctx.stores_vec { - // Multi-store: resolve chunks across all stores - let dep_aliases = ctx.aliases(); - let mut seen_paths = HashSet::new(); - let mut out = Vec::new(); - for f in &fts_results { - for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.get_chunk(f.chunk_id) { - Ok(Some(chunk)) => { - if !is_import_kind(&chunk.kind) { - break; // try next FTS result - } - - let norm = crate::cache::normalize_path_str(&chunk.path); - if !seen_paths.insert(norm) { - break; - } - - let term_lower = search_term.to_lowercase(); - let import_statement = - if chunk.content.to_lowercase().contains(&term_lower) { - chunk - .content - .lines() - .find(|l| l.to_lowercase().contains(&term_lower)) - .unwrap_or("") - .to_string() - } else { - chunk.signature.filter(|s| !s.is_empty()).unwrap_or( - chunk.content.lines().next().unwrap_or("").to_string(), - ) - }; - - out.push(DependentItem { - path: chunk.path, - line: chunk.start_line, - import_statement, - }); - - break; // found in this store, move to next FTS result - } - Ok(None) => {} // try next store - // One broken store says nothing about the others; a - // `break` here silently drops a chunk that lives in a - // healthy store later in the list. - Err(ref e) => { - note_store_failure( - &mut dep_warnings, - dep_aliases, - store_idx, - "chunk lookup", - e, - ); - continue; - } - } - } - if out.len() >= limit { - break; - } - } - out - } else { - match self - .with_vector_store_read_for( - |store| { - let mut seen_paths = HashSet::new(); - let mut out = Vec::new(); - let term_lower = search_term.to_lowercase(); - for f in &fts_results { - if let Some(chunk) = store.get_chunk(f.chunk_id)? { - if !is_import_kind(&chunk.kind) { - continue; - } - - let norm = crate::cache::normalize_path_str(&chunk.path); - if !seen_paths.insert(norm) { - continue; - } - - // Extract the specific import line(s) that mention the - // module name, rather than returning the entire chunk content. - let import_statement = - if chunk.content.to_lowercase().contains(&term_lower) { - chunk - .content - .lines() - .find(|l| l.to_lowercase().contains(&term_lower)) - .unwrap_or("") - .to_string() - } else { - chunk.signature.filter(|s| !s.is_empty()).unwrap_or( - chunk.content.lines().next().unwrap_or("").to_string(), - ) - }; - - out.push(DependentItem { - path: chunk.path, - line: chunk.start_line, - import_statement, - }); - - if out.len() >= limit { - break; - } - } - } - Ok(out) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error resolving dependents: {e:#}" - ))])); - } - } - }; - - // Prefix paths with alias for multi-repo identification - for item in &mut items { - item.path = ctx.prefix_result_path(&item.path); - } - - items.sort_by(|a, b| a.path.cmp(&b.path)); - respond_with_items(&items, &dep_warnings, || { - format!("No dependent files found for '{}'.", request.symbol_or_path) - }) - } - - /// Internal: find similar chunks, used by `explore(kind="similar")`. - async fn similar_chunks( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "explore") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - let limit = request.limit.unwrap_or(5).min(20); - - // Stores that failed while resolving the source embedding. `if let - // Ok(Some(..))` used to discard the error, so a dead store produced - // "embedding not found" — a wrong diagnosis, not a missing chunk. - let mut similar_warnings: Vec = Vec::new(); - - let mut results = if let Some(ref sv) = ctx.stores_vec { - // Multi-store: find the embedding in whichever store has it, - // then search across all stores for similar chunks. - let aliases = ctx.aliases(); - let mut embedding: Option> = None; - for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.get_embedding(request.chunk_id) { - Ok(Some(emb)) => { - embedding = Some(emb); - break; - } - Ok(None) => continue, - Err(ref e) => { - note_store_failure( - &mut similar_warnings, - aliases, - i, - "embedding lookup", - e, - ); - continue; - } - } - } - - let embedding = match embedding { - Some(e) => e, - None => { - return Ok(CallToolResult::success(vec![Content::text( - qualify_empty_result( - format!( - "Embedding not found for chunk_id {} in any store.", - request.chunk_id - ), - &similar_warnings, - ), - )])); - } - }; - - // Search across all stores with the found embedding - let mut all_results: Vec = Vec::new(); - let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); - for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.search(&embedding, limit + 1) { - Ok(mut neighbors) => { - neighbors.retain(|r| r.id != request.chunk_id); - for r in neighbors { - if seen_ids.insert(r.id) { - all_results.push(SearchResultItem { - chunk_id: r.id, - path: r.path, - start_line: r.start_line, - end_line: r.end_line, - kind: r.kind, - score: r.score, - signature: r.signature, - content: None, - context_prev: None, - context_next: None, - source: None, - chunk_ref: None, - }); - } - } - } - Err(ref e) => { - // The embedding was found, so the handler returns results - // either way; without this, a group query silently omits - // every neighbour from the broken repo. - note_store_failure( - &mut similar_warnings, - aliases, - store_idx, - "similarity search", - e, - ); - } - } - } - - all_results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - all_results.truncate(limit); - all_results - } else { - match self - .with_vector_store_read_for( - |store| { - let embedding = - store.get_embedding(request.chunk_id)?.ok_or_else(|| { - anyhow::anyhow!( - "embedding not found for chunk_id {}", - request.chunk_id - ) - })?; - - let mut neighbors = store.search(&embedding, limit + 1)?; - neighbors.retain(|r| r.id != request.chunk_id); - neighbors.truncate(limit); - - let items = neighbors - .into_iter() - .map(|r| SearchResultItem { - chunk_id: r.id, - path: r.path, - start_line: r.start_line, - end_line: r.end_line, - kind: r.kind, - score: r.score, - signature: r.signature, - content: None, - context_prev: None, - context_next: None, - source: None, - chunk_ref: None, - }) - .collect::>(); - Ok(items) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error finding similar chunks: {e:#}" - ))])); - } - } - }; - - // Prefix paths with alias for multi-repo identification - for item in &mut results { - item.path = ctx.prefix_result_path(&item.path); - } - - // Every exit carries the channel: the earlier read sat in an - // early-return arm, so once an embedding was found, every failure - // recorded afterwards (the whole neighbour fan-out) was discarded. - respond_with_items(&results, &similar_warnings, || { - format!("No similar chunks found for chunk_id {}.", request.chunk_id) - }) - } - - async fn literal_search( - &self, - Parameters(request): Parameters, - ) -> Result { - // Resolve project/group routing - let ctx = match self - .resolve_routing(&request.project, &request.group, false, "search") - .await - { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - let limit = request.limit.unwrap_or(20); - let output_format = request.format.as_deref().unwrap_or("json"); - - // Repos that failed during this search. Reported to the caller: an - // agent that never sees the server log cannot otherwise distinguish a - // broken store from a repo that holds no match. - let mut literal_warnings: Vec = Vec::new(); - - // Auto-regex promotion: detect code patterns that BM25 would destroy - let user_set_regex = request.regex.unwrap_or(false); - let user_set_phrase = request.phrase.unwrap_or(false); - let auto_promoted = - !user_set_regex && !user_set_phrase && looks_like_code_pattern(&request.query); - - let (effective_query, effective_regex) = if auto_promoted { - let escaped = regex::escape(&request.query); - // Relax whitespace to \s+ so "foo = null" → "foo\s+=\s+null" - // regex::escape does not escape spaces, so replace literal spaces. - let relaxed = escaped.replace(' ', r"\s+"); - (relaxed, true) - } else { - (request.query.clone(), user_set_regex) - }; - - tracing::debug!( - "MCP literal_search: query='{}', regex={:?}, phrase={:?}, limit={}, file_glob={:?}, language={:?}, format={}, multi={}", - request.query, request.regex, request.phrase, limit, - request.file_glob, request.language, output_format, ctx.is_multi - ); - - if ctx.needs_local_db { - if let Err(e) = self.ensure_database_exists() { - return Ok(CallToolResult::success(vec![Content::text(e)])); - } - } - - // Pre-compute normalized project root for stripping absolute paths in glob matching - let lang_filter = request.language.clone(); - let glob_filter = request.file_glob.clone(); - let regex_enabled = effective_regex; - let snippet_regex = if regex_enabled { - Regex::new(&effective_query).ok() - } else { - None - }; - let project_root_normalized = { - let root = crate::cache::normalize_path_str(self.project_path.to_str().unwrap_or("")); - root.trim_end_matches('/').to_string() - }; - - // Decide: BM25 path (for anchorable queries) or scan path (for tokenless regex - // or disjunctive OR patterns like TODO|FIXME|HACK that BM25 treats as AND). - let tokenless_regex = regex_enabled - && snippet_regex.is_some() - && (!regex_has_anchorable_token(&effective_query) - || regex_has_disjunctive_or(&effective_query)); - - let mut items: Vec = if tokenless_regex { - // ── Scan path ────────────────────────────────────────────── - // Tokenless regex (e.g. \bfn\s+\w+) — BM25 cannot produce useful - // candidates. Scan all chunks sequentially, apply regex post-filter. - // Score is 0.0 for all results (no BM25 ranking applies). - tracing::debug!("literal_search: tokenless regex detected, using scan path"); - if let Some(ref sv) = ctx.stores_vec { - // Multi-store scan - let mut items: Vec = Vec::new(); - for store_arc in sv { - let store = store_arc.vector_store.read().await; - let all_chunks = match store.iter_all_chunks() { - Ok(chunks) => chunks, - Err(_) => continue, - }; - for (_, chunk) in all_chunks { - if let Some(ref lang) = lang_filter { - let file_lang = Language::from_path(std::path::Path::new(&chunk.path)); - if file_lang.name() != lang { - continue; - } - } - if let Some(ref glob) = glob_filter { - let relative_path = chunk - .path - .strip_prefix(&project_root_normalized) - .unwrap_or(&chunk.path) - .trim_start_matches('/'); - if !simple_glob_match(glob, relative_path) { - continue; - } - } - if let Some((match_offset, snippet)) = match_line_for_literal( - &chunk.content, - &effective_query, - snippet_regex.as_ref(), - ) { - let match_line = chunk.start_line + match_offset; - items.push(LiteralSearchResultItem { - path: chunk.path, - start_line: match_line, - end_line: match_line, - snippet, - score: 0.0, // No BM25 score — scan-path results are unranked - kind: if chunk.kind.is_empty() { - None - } else { - Some(chunk.kind) - }, - signature: chunk.signature.filter(|s| !s.is_empty()), - }); - if items.len() >= limit { - break; - } - } - } - if items.len() >= limit { - break; - } - } - items - } else { - // Single-store scan - match self - .with_vector_store_read_for( - |store| { - let all_chunks = store.iter_all_chunks()?; - let mut items: Vec = Vec::new(); - for (_, chunk) in all_chunks { - if let Some(ref lang) = lang_filter { - let file_lang = - Language::from_path(std::path::Path::new(&chunk.path)); - if file_lang.name() != lang { - continue; - } - } - if let Some(ref glob) = glob_filter { - let relative_path = chunk - .path - .strip_prefix(&project_root_normalized) - .unwrap_or(&chunk.path) - .trim_start_matches('/'); - if !simple_glob_match(glob, relative_path) { - continue; - } - } - if let Some((match_offset, snippet)) = match_line_for_literal( - &chunk.content, - &effective_query, - snippet_regex.as_ref(), - ) { - let match_line = chunk.start_line + match_offset; - items.push(LiteralSearchResultItem { - path: chunk.path, - start_line: match_line, - end_line: match_line, - snippet, - score: 0.0, // No BM25 score — scan-path results are unranked - kind: if chunk.kind.is_empty() { - None - } else { - Some(chunk.kind) - }, - signature: chunk.signature.filter(|s| !s.is_empty()), - }); - if items.len() >= limit { - break; - } - } - } - Ok(items) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error scanning chunks: {e:#}" - ))])); - } - } - } - } else { - // ── BM25 path ────────────────────────────────────────────── - // Note: regex=true uses BM25 for candidates, then post-filters with the - // actual regex on raw content (Tantivy's RegexQuery only works on individual - // tokens, not raw text — underscores/punctuation cause empty results). - // - // When regex is enabled, strip metacharacters from the BM25 query so - // Tantivy gets clean tokens (e.g. "class Cache" instead of "class \w+Cache\b"). - let bm25_query = if regex_enabled { - let cleaned = extract_bm25_query_from_regex(&effective_query); - if cleaned.is_empty() { - effective_query.clone() - } else { - cleaned - } - } else { - effective_query.clone() - }; - let fts_results = if let Some(ref sv) = ctx.stores_vec { - let sa = ctx.store_aliases.as_ref().unwrap(); - let outcome = self - .with_fts_store_read_multi( - |fts_store| { - if request.phrase.unwrap_or(false) { - fts_store.search_phrase(&bm25_query, limit * 3) - } else { - fts_store.search(&bm25_query, limit * 3, None) - } - }, - sv.clone(), - sa, - ) - .await - .unwrap_or_default(); - for (alias, err) in &outcome.failures { - let msg = format!("repo '{alias}' literal search failed: {err}"); - tracing::error!("MCP: {}", msg); - literal_warnings.push(msg); - } - outcome.results - } else { - match self - .with_fts_store_read_for( - |fts_store| { - if request.phrase.unwrap_or(false) { - fts_store.search_phrase(&bm25_query, limit * 3) - } else { - fts_store.search(&bm25_query, limit * 3, None) - } - }, - ctx.stores.clone(), - ) - .await - { - Ok(r) => r, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error searching: {e:#}" - ))])); - } - } - }; - - // Resolve chunk metadata and apply post-filters - if let Some(ref sv) = ctx.stores_vec { - // Multi-store: resolve chunks from all stores - let mut items: Vec = Vec::new(); - 'outer: for fts_result in &fts_results { - let sa = ctx.store_aliases.as_ref().unwrap(); - for (idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - let looked_up = store.get_chunk(fts_result.chunk_id); - if let Err(ref e) = looked_up { - note_store_failure(&mut literal_warnings, sa, idx, "chunk lookup", e); - } - if let Some(chunk) = looked_up.ok().flatten() { - if let Some(ref lang) = lang_filter { - let file_lang = - Language::from_path(std::path::Path::new(&chunk.path)); - if file_lang.name() != lang { - continue; - } - } - if let Some(ref glob) = glob_filter { - let relative_path = chunk - .path - .strip_prefix(&project_root_normalized) - .unwrap_or(&chunk.path) - .trim_start_matches('/'); - if !simple_glob_match(glob, relative_path) { - continue; - } - } - let match_info = match_line_for_literal( - &chunk.content, - &effective_query, - snippet_regex.as_ref(), - ); - if regex_enabled && match_info.is_none() { - continue; - } - let (match_offset, snippet) = match_info.unwrap_or_else(|| { - (0, chunk.content.lines().next().unwrap_or("").to_string()) - }); - let match_line = chunk.start_line + match_offset; - items.push(LiteralSearchResultItem { - path: chunk.path, - start_line: match_line, - end_line: match_line, - snippet, - score: fts_result.score, - kind: if chunk.kind.is_empty() { - None - } else { - Some(chunk.kind) - }, - signature: chunk.signature.filter(|s| !s.is_empty()), - }); - if items.len() >= limit { - break 'outer; - } - break; // Found in this store - } - } - } - items - } else { - match self - .with_vector_store_read_for( - |store| { - let items: Vec = fts_results - .iter() - .filter_map(|fts_result| { - let chunk = store.get_chunk(fts_result.chunk_id).ok()??; - Some((chunk, fts_result.score)) - }) - .filter(|(chunk, _)| { - if let Some(ref lang) = lang_filter { - let file_lang = - Language::from_path(std::path::Path::new(&chunk.path)); - if file_lang.name() != lang { - return false; - } - } - if let Some(ref glob) = glob_filter { - let relative_path = chunk - .path - .strip_prefix(&project_root_normalized) - .unwrap_or(&chunk.path) - .trim_start_matches('/'); - if !simple_glob_match(glob, relative_path) { - return false; - } - } - true - }) - .take(limit) - .filter_map(|(chunk, score)| { - let match_info = match_line_for_literal( - &chunk.content, - &effective_query, - snippet_regex.as_ref(), - ); - if regex_enabled && match_info.is_none() { - return None; - } - let (match_offset, snippet) = match_info.unwrap_or_else(|| { - (0, chunk.content.lines().next().unwrap_or("").to_string()) - }); - let match_line = chunk.start_line + match_offset; - Some(LiteralSearchResultItem { - path: chunk.path, - start_line: match_line, - end_line: match_line, - snippet, - score, - kind: if chunk.kind.is_empty() { - None - } else { - Some(chunk.kind) - }, - signature: chunk.signature.filter(|s| !s.is_empty()), - }) - }) - .collect(); - Ok(items) - }, - ctx.stores.clone(), - ) - .await - { - Ok(items) => items, - Err(e) => { - return Ok(CallToolResult::success(vec![Content::text(format!( - "Error resolving search results: {e:#}" - ))])); - } - } - } - }; - - // Prefix paths with alias for multi-repo identification - for item in &mut items { - item.path = ctx.prefix_result_path(&item.path); - } - - // Compute low-confidence signal - let top_score = items.first().map(|i| i.score); - let (low_confidence, suggested_tool) = - compute_literal_low_confidence(top_score, &request.query); - - // Build note - let note = if auto_promoted { - Some(format!( - "Query auto-promoted to regex mode (original: '{}', effective: '{}'). \ - The query contained code-like punctuation that BM25 would tokenize incorrectly.", - request.query, effective_query - )) - } else if low_confidence == Some(true) { - suggested_tool.as_ref().map(|tool| { - format!( - "Top result has weak BM25 score; consider using `{}` for better matches.", - tool - ) - }) - } else { - None - }; - - let response = LiteralSearchResponse { - results: items, - auto_promoted_to_regex: if auto_promoted { Some(true) } else { None }, - note, - low_confidence, - suggested_tool: if low_confidence == Some(true) { - suggested_tool - } else { - None - }, - warnings: if literal_warnings.is_empty() { - None - } else { - Some(literal_warnings) - }, - }; - - // Instrument BM25 score for threshold calibration - if let Some(top) = response.results.first() { - tracing::debug!( - target: "codesearch::literal_confidence", - query = %request.query, - top_bm25_score = top.score, - result_count = response.results.len(), - "literal_search score sample" - ); - } - - // Format output - let output = if output_format == "grep" { - let mut lines: Vec = Vec::new(); - if response.auto_promoted_to_regex == Some(true) { - lines.push( - "# auto-promoted to regex mode (query contained code-like punctuation)" - .to_string(), - ); - } - if response.low_confidence == Some(true) { - if let Some(ref hint) = response.suggested_tool { - lines.push(format!("# low confidence — consider: {}", hint)); - } - } - for item in &response.results { - lines.push(format!( - "{}:{}:{}", - item.path, item.start_line, item.snippet - )); - } - lines.join("\n") - } else { - serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()) - }; - - Ok(CallToolResult::success(vec![Content::text(output)])) - } - - /// Internal implementation for index_status with optional project/group routing. - async fn index_status_impl( - &self, - project: Option, - group: Option, - ) -> Result { - // When no project/group specified in serve mode, return lightweight aggregated - // status WITHOUT opening any databases. Only a specific project/group request - // should trigger DB activation. - if project.is_none() && group.is_none() { - if let Some(ref serve_state) = self.serve_state { - let config = serve_state.config_snapshot(); - let repo_count = config.repos.len(); - // Count the virtual "all" group when repos are registered, so the - // summary doesn't read "0 group(s)" while `all` is actually available. - let group_count = config.groups.len() + if config.repos.is_empty() { 0 } else { 1 }; - let statuses = serve_state.repo_statuses_lightweight(); - let open_count = statuses - .iter() - .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Open)) - .count(); - let warm_count = statuses - .iter() - .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Warm)) - .count(); - let closed_count = statuses - .iter() - .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Closed)) - .count(); - - let status = if open_count + warm_count > 0 { - "ready".to_string() - } else if repo_count > 0 { - "idle".to_string() - } else { - "no_repos".to_string() - }; - - let status_message = format!( - "{} repo(s) registered, {} group(s). Open: {}, Warm: {}, Closed: {}.", - repo_count, group_count, open_count, warm_count, closed_count - ); - - let response = IndexStatusResponse { - indexed: open_count + warm_count > 0, - status, - status_message, - total_chunks: 0, // Not available without opening DBs - total_files: 0, - model: self.model_type.short_name().to_string(), - dimensions: 0, - max_chunk_id: 0, - db_path: format!("({} repos)", repo_count), - project_path: format!("serve mode — {} repo(s)", repo_count), - error_message: None, - mode: self.mcp_mode(), - }; - - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - return Ok(CallToolResult::success(vec![Content::text(json)])); - } - } - - // Resolve project/group routing — status is scope-free, allow unscoped fan-out - let ctx = match self.resolve_routing(&project, &group, true, "status").await { - Ok(c) => c, - Err(e) => return Ok(CallToolResult::success(vec![Content::text(e)])), - }; - - if ctx.needs_local_db { - let indexed = self.db_path.exists(); - - if !indexed { - let response = IndexStatusResponse { - indexed: false, - status: "not_indexed".to_string(), - status_message: "No index found. Run 'codesearch index' or start with --create-index=true to automatically create one.".to_string(), - total_chunks: 0, - total_files: 0, - model: "none".to_string(), - dimensions: 0, - max_chunk_id: 0, - db_path: self.db_path.display().to_string(), - project_path: self.project_path.display().to_string(), - error_message: None, - mode: self.mcp_mode(), - }; - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - return Ok(CallToolResult::success(vec![Content::text(json)])); - } - } - - if let Some(ref sv) = ctx.stores_vec { - // Multi-store: aggregate stats across all group members - let mut total_chunks = 0usize; - let mut total_files = 0usize; - let mut max_chunk_id = 0u32; - let mut dimensions = 0usize; - let mut all_indexed = true; - let aliases = ctx.aliases(); - let mut stats_warnings: Vec = Vec::new(); - let mut failed_count = 0usize; - - for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.stats() { - Ok(stats) => { - total_chunks += stats.total_chunks; - total_files += stats.total_files; - if stats.max_chunk_id > max_chunk_id { - max_chunk_id = stats.max_chunk_id; - } - if stats.dimensions > 0 { - dimensions = stats.dimensions; - } - if !stats.indexed { - all_indexed = false; - } - } - // `all_indexed = false` alone renders identically to "still - // warming" — the caller has no way to tell "wait" from "this - // store is down". This is the tool whose job is reporting index - // health, so it must not stay silent on the one signal that - // matters here: bind the error, carry it, never `Err(_)`. - Err(ref e) => { - all_indexed = false; - failed_count += 1; - note_store_failure(&mut stats_warnings, aliases, i, "stats", e); - } - } - } - - let (status, status_message) = - index_status_summary(sv.len(), failed_count, total_chunks); - - let response = IndexStatusResponse { - indexed: all_indexed, - status, - status_message, - total_chunks, - total_files, - model: self.model_type.short_name().to_string(), - dimensions, - max_chunk_id, - db_path: format!("({} repos)", sv.len()), - project_path: format!("group with {} repo(s)", sv.len()), - error_message: None, - mode: self.mcp_mode(), - }; - - return respond_with_object(&response, &stats_warnings); - } - - // Single-store path - let stats = match self - .with_vector_store_read_for( - |store| store.stats().context("Error getting index stats"), - ctx.stores.clone(), - ) - .await - { - Ok(s) => s, - Err(e) => { - let response = IndexStatusResponse { - indexed: false, - status: "error".to_string(), - status_message: format!("{}", e), - total_chunks: 0, - total_files: 0, - model: self.model_type.short_name().to_string(), - dimensions: 0, - max_chunk_id: 0, - db_path: self.db_path.display().to_string(), - project_path: self.project_path.display().to_string(), - error_message: Some(format!("{}", e)), - mode: self.mcp_mode(), - }; - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - return Ok(CallToolResult::success(vec![Content::text(json)])); - } - }; - - // Determine status based on database state - let (status, status_message) = if stats.total_chunks == 0 { - ( - "building".to_string(), - "Index is being built in the background. Searches may fail until indexing completes. Please check back in a few minutes.".to_string(), - ) - } else { - ( - "ready".to_string(), - "Index is ready for searching.".to_string(), - ) - }; - - let response = IndexStatusResponse { - indexed: stats.indexed, - status, - status_message, - total_chunks: stats.total_chunks, - total_files: stats.total_files, - model: self.model_type.short_name().to_string(), - dimensions: stats.dimensions, - max_chunk_id: stats.max_chunk_id, - db_path: self.db_path.display().to_string(), - project_path: self.project_path.display().to_string(), - error_message: None, - mode: self.mcp_mode(), - }; - - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) - } - - /// List all registered projects and groups. Called by `status(kind="projects")`. - /// Build the `remote_projects` listing (opt-in mounts) for `list_projects`. - fn remote_projects_listing( - config: &crate::db_discovery::repos::ReposConfig, - ) -> Vec { - config - .mounted_remote_projects() - .into_iter() - .filter_map(|(name, target)| match target { - crate::db_discovery::repos::Target::RemoteProject { - peer_name, - peer, - remote_alias, - } => Some(RemoteProjectInfo { - name, - peer: peer_name, - remote_alias, - peer_url: peer.url, - }), - _ => None, - }) - .collect() - } - - async fn list_projects(&self) -> Result { - let current_dir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - - let serve_active = self.serve_state.is_some(); - let serve_url = if serve_active { - Some(serve_url_from_env()) - } else { - None - }; - - // When serve is active, use ServeState as source of truth for lock status - if let Some(ref serve_state) = self.serve_state { - let config = serve_state.config_snapshot(); - let project_groups = config.project_groups(); - let mut repos_info = Vec::new(); - let mut list_warnings: Vec = Vec::new(); - - for (alias, path) in &config.repos { - let db_path = path.join(crate::constants::DB_DIR_NAME); - - let (total_chunks, total_files, model, lock_status, error) = if db_path.exists() { - let (model_name, _dims) = read_model_metadata(&db_path); - - // For repos already opened in DashMap, use the live SharedStores for stats - // WITHOUT opening a new VectorStore connection. - // For unopened repos, just report metadata — do NOT open the DB. - if let Some(stores) = serve_state.get_opened_stores(alias) { - let stats_result = { - let vs = stores.vector_store.read().await; - vs.stats() - }; - // `0 chunks` alone reads exactly like "not indexed yet" — the - // repo may in fact be full and simply failing to answer (the - // read-only-incident shape this branch exists for). Attribute - // the failure to THIS repo rather than a top-level channel: - // list_projects returns one entry per repo, so per-item is the - // shape that actually matches the fan-out. - // `repo_stats_from_result` carries only the part of this - // decision that varies by Ok/Err — see its doc comment. - // `record_stats_or_warn` wraps it so this call site cannot - // silently drop the warning half without also breaking the - // counts it returns — see its own doc comment. - let (total_chunks, total_files, error) = - record_stats_or_warn(stats_result, alias, &mut list_warnings); - ( - total_chunks, - total_files, - model_name, - serve_state - .repo_lock_status(alias) - .unwrap_or("unknown") - .to_string(), - error, - ) - } else { - // Repo NOT opened — read persisted stats from metadata.json - let (md_chunks, md_files) = read_metadata_stats(&db_path); - let lock_status = if crate::index::is_database_locked(&db_path) { - "locked-externally".to_string() - } else { - "available".to_string() - }; - (md_chunks, md_files, model_name, lock_status, None) - } - } else { - (0, 0, "not indexed".to_string(), "unknown".to_string(), None) - }; - - repos_info.push(RepoInfo { - alias: alias.clone(), - project_path: path.display().to_string(), - database_path: db_path.display().to_string(), - total_chunks, - total_files, - model, - lock_status, - groups: project_groups.get(alias).cloned().unwrap_or_default(), - error, - }); - } - - let response = ListProjectsResponse { - repos: repos_info, - groups: config.groups_with_virtual_all(), - remote_projects: Self::remote_projects_listing(&config), - serve_active, - serve_url, - current_directory: current_dir.display().to_string(), - }; - - return respond_with_object(&response, &list_warnings); - } - - // Stdio mode: fall back to disk-based lock detection - let config = load_repos_config().unwrap_or_default(); - let project_groups = config.project_groups(); - let mut repos_info = Vec::new(); - for (alias, path) in &config.repos { - let db_path = path.join(crate::constants::DB_DIR_NAME); - - // Get stats - let (total_chunks, total_files, model, lock_status) = if db_path.exists() { - let (model_name, dims) = read_model_metadata(&db_path); - - let lock = if crate::index::is_database_locked(&db_path) { - "conflicted" - } else { - "available" - }; - - if let Ok(store) = VectorStore::new(&db_path, dims) { - if let Ok(stats) = store.stats() { - ( - stats.total_chunks, - stats.total_files, - model_name, - lock.to_string(), - ) - } else { - (0, 0, model_name, lock.to_string()) - } - } else { - (0, 0, model_name, "readonly".to_string()) - } - } else { - (0, 0, "not indexed".to_string(), "unknown".to_string()) - }; - - // Stdio mode is single-repo-at-a-time CLI usage, not the live multi-repo - // federation this fan-out fix targets — a stats() failure here is out of - // scope for this fix (VectorStore::new/stats failing locally is a different - // shape than a store going down mid-request in a shared serve process). - repos_info.push(RepoInfo { - alias: alias.clone(), - project_path: path.display().to_string(), - database_path: db_path.display().to_string(), - total_chunks, - total_files, - model, - lock_status, - groups: project_groups.get(alias).cloned().unwrap_or_default(), - error: None, - }); - } - - let response = ListProjectsResponse { - repos: repos_info, - groups: config.groups_with_virtual_all(), - remote_projects: Self::remote_projects_listing(&config), - serve_active, - serve_url, - current_directory: current_dir.display().to_string(), - }; - - let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); - Ok(CallToolResult::success(vec![Content::text(json)])) - } -} - -// === Server Handler Implementation === - -/// Check if a chunk is a definition of the given symbol. -/// -/// Best-effort heuristic for v1: a chunk is considered a definition if: -/// 1. Its kind is a definition kind (Function, Struct, Class, etc.) -/// 2. Its signature starts with a common definition pattern containing the symbol name -/// -/// Limitation: this uses simple substring matching on the signature field. -/// False positives/negatives are possible for symbols that appear in signatures -/// of chunks that are not their definitions. -fn is_definition_chunk(kind: &str, signature: &Option, symbol: &str) -> bool { - // Only check definition kinds - if !DEFINITION_KINDS.contains(&kind) { - return false; - } - - let sig = match signature { - Some(s) if !s.is_empty() => s, - _ => return false, - }; - - // Common definition prefixes across languages. - // Keep this allocation-free in hot paths by using &str prefixes and boundary checks. - const PREFIXES: &[&str] = &[ - "fn ", - "def ", - "class ", - "struct ", - "enum ", - "trait ", - "type ", - "interface ", - "impl ", - "pub fn ", - "pub async fn ", - "pub struct ", - "pub enum ", - "pub trait ", - "pub type ", - "async fn ", - "const ", - "static ", - ]; - - let prefix_match = PREFIXES.iter().any(|prefix| { - if !sig.starts_with(prefix) { - return false; - } - - let rest = &sig[prefix.len()..]; - if !rest.starts_with(symbol) { - return false; - } - - let next = rest[symbol.len()..].chars().next(); - matches!(next, None | Some('(' | '<' | ':' | ' ' | '\t')) - }); - - if prefix_match { - return true; - } - - // Fallback for languages with verbose signatures (C#, Java): - // signatures include access modifiers and return types before the symbol name, - // e.g. "public async Task UploadFileAsync(...)" or "protected override void Update(...)". - // Search for the symbol as a whole word anywhere in the signature. - contains_symbol_as_word(sig, symbol) -} - -/// Check whether `symbol` appears as a whole word in `sig`. -/// A word boundary requires the character before to be a space/tab (or start-of-string) -/// and the character after to be `(`, `<`, `:`, space, tab, or end-of-string. -/// This is intentionally conservative to avoid matching parameter type names. -fn contains_symbol_as_word(sig: &str, symbol: &str) -> bool { - let sig_bytes = sig.as_bytes(); - let sym_len = symbol.len(); - let mut start = 0usize; - while start + sym_len <= sig.len() { - if let Some(rel) = sig[start..].find(symbol) { - let abs = start + rel; - let before_ok = abs == 0 - || matches!( - sig_bytes.get(abs - 1), - Some(&b' ') | Some(&b'\t') | Some(&b'\n') - ); - let after_char = sig[abs + sym_len..].chars().next(); - let after_ok = matches!(after_char, None | Some('(' | '<' | ':' | ' ' | '\t')); - if before_ok && after_ok { - return true; - } - start = abs + 1; - } else { - break; - } - } - false -} - -/// MCP server instructions template (pre-substitution). Kept as a named const so -/// the line-count and deprecated-alias tests can validate it directly without -/// fragile `include_str!` source-text searching or instantiating the service. -/// Substitution uses `str::replace` (not `format!`) because `format!` requires a -/// literal; the placeholders are unique tokens that don't appear in the prose. -/// See `test_instructions_max_50_lines` / `test_no_deprecated_tool_aliases`. -const INSTRUCTIONS_TEMPLATE: &str = r#"codesearch — semantic code search + symbol impact analysis. - -WHEN TO USE codesearch (prefer over grep/glob): - Good for: semantic or cross-file lookup, unknown file paths, symbol navigation, - "where is X implemented", "find usages of Y", "how does Z flow through the code" - Not for: a single known file (just read it), trivial one-line edits, - exact literal patterns where plain grep is faster - -SERVICE-MODE NOTES (codesearch serve, esp. on another host): - - Paths come from the SERVER's filesystem. Use get_chunk to read content; - don't try to open returned paths locally. - - Not every directory is indexed (e.g. .venv, node_modules, build/). If a - search returns nothing, the dir may be unindexed — ask, don't grep blindly. - -PICK THE RIGHT TOOL FOR THE TASK: - "who calls X?" / "what breaks if I rename X?" - → find_impact (precise SCIP call-graph; if no backend for the language, it says so → then use find kind="usages") - "find code about X" / "how does X work" / "show me X" - → search(mode="semantic") — concepts + synonyms + identifiers - exact syntax like Vec / foo = null / a::b - → search(mode="literal", regex=true) — patterns semantic can't match - "where is X defined?" / "what does file X import?" - → find(kind="definition" | "imports") - "show all symbols in file X" / "code like chunk Y" - → explore(kind="outline" | "similar") - read chunk content → get_chunk(chunk_id) - index health / repo list → status - -RULES: - - search(semantic) is the DEFAULT for code lookup. Don't skip it. - - For "who calls X" / impact analysis, try find_impact first; fall back to find(kind="usages") only if find_impact reports no backend. - - NEVER use literal as first search unless you need exact syntax. - - project or group is REQUIRED in multi-repo mode. - -Mode: {mode} -Project: {project} -Database: {db} ({exists}) -Model: {model} ({dims}d) -"#; - -// ════════════════════════════════════════════════════════════════ -// Federation helpers (module-scope) — merge / parse / convert. -// ════════════════════════════════════════════════════════════════ - -/// RRF-interleave several disjoint ranked lists into one ranked list. -/// -/// Each list is assumed already ranked best-first and disjoint from the others -/// (local repos vs. distinct remote peers). An item's merged score is -/// `1/(k + rank_in_own_list + 1)` (classic Reciprocal Rank Fusion with a `+1` -/// so the top hit never exceeds `1/k`). The union is sorted by score desc with a -/// stable source-order tiebreak, then truncated to `limit`. -fn merge_ranked_lists( - lists: Vec>, - k: f32, - limit: usize, -) -> Vec { - let mut merged: Vec<(f32, usize, SearchResultItem)> = Vec::new(); - let mut order = 0usize; - for list in lists { - for (rank, item) in list.into_iter().enumerate() { - let score = 1.0 / (k + rank as f32 + 1.0); - merged.push((score, order, item)); - order += 1; - } - } - // Sort by score desc; tiebreak on insertion order for stable, predictable - // output (local list first, then remotes in config order). - merged.sort_by(|a, b| { - b.0.partial_cmp(&a.0) - .unwrap_or(std::cmp::Ordering::Equal) - .then(a.1.cmp(&b.1)) - }); - merged - .into_iter() - .take(limit) - .map(|(score, _, mut it)| { - it.score = score; - it - }) - .collect() -} - -/// Extract the rendered tool payload from a `CallToolResult` and re-parse it as -/// local `SearchResultItem`s. Works for both semantic and literal modes — the -/// rendered JSON always has a top-level `results` array. -fn parse_search_items_from_call_result( - result: &CallToolResult, - mode: &str, -) -> Vec { - let text = extract_call_tool_text(result); - let value: serde_json::Value = match serde_json::from_str(&text) { - Ok(v) => v, - Err(_) => return Vec::new(), - }; - let results = match value.get("results").and_then(|r| r.as_array()) { - Some(arr) => arr, - None => return Vec::new(), - }; - match mode { - "semantic" => results - .iter() - .filter_map(|v| serde_json::from_value::(v.clone()).ok()) - .collect(), - // Literal items lack `chunk_id`; map their `snippet` into `content` so - // the merged list renders uniformly. - _ => results - .iter() - .map(|v| SearchResultItem { - chunk_id: v.get("chunk_id").and_then(|c| c.as_u64()).unwrap_or(0) as u32, - path: v - .get("path") - .and_then(|p| p.as_str()) - .unwrap_or("") - .to_string(), - start_line: v.get("start_line").and_then(|n| n.as_u64()).unwrap_or(0) as usize, - end_line: v.get("end_line").and_then(|n| n.as_u64()).unwrap_or(0) as usize, - kind: v - .get("kind") - .and_then(|k| k.as_str()) - .unwrap_or("") - .to_string(), - score: v.get("score").and_then(|s| s.as_f64()).unwrap_or(0.0) as f32, - signature: v - .get("signature") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()), - content: v - .get("snippet") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()), - context_prev: None, - context_next: None, - source: None, - chunk_ref: None, - }) - .collect(), - } -} - -/// Convert a remote search hit into a local `SearchResultItem`, tagging it with -/// its origin (`source`) and a project-namespaced `chunk_ref` for later -/// retrieval. -/// -/// The `chunk_ref` is `"/:"`. The `remote_alias` -/// segment is essential: the peer is itself multi-repo and chunk_ids are only -/// unique *within* a single index, so `federated_get_chunk` must forward the -/// alias as a `project=` scope to disambiguate. Omitting it (the old -/// `":"` shape) made every remote `get_chunk` fail with -/// `ambiguous_chunk_id` whenever the peer hosted more than one project. -fn convert_remote_item( - peer_name: &str, - remote_alias: &str, - item: crate::federation::RemoteSearchItem, -) -> SearchResultItem { - let chunk_ref = item - .chunk_id - .map(|id| format!("{peer_name}/{remote_alias}:{id}")); - SearchResultItem { - chunk_id: item.chunk_id.unwrap_or(0), - path: item.path, - start_line: item.start_line, - end_line: item.end_line, - kind: item.kind.unwrap_or_default(), - score: item.score, - signature: item.signature, - content: item.content.or(item.snippet), - context_prev: item.context_prev, - context_next: item.context_next, - source: Some(format!("{peer_name}/{remote_alias}")), - chunk_ref, - } -} - -/// Apply a `filter_path` prefix filter to federated results **client-side**, -/// on the namespaced paths the caller actually sees. -/// -/// Federated `filter_path` cannot be forwarded to the peer: the peer matches -/// against its own un-namespaced store paths (and, in serve mode, against the -/// wrong project root), so a server-side match returns nothing for any value. -/// Here we match against the `//…` path carried on each converted -/// item, with an empty project root (the namespaced path is already relative), -/// so the filter means exactly what the caller reads back in the results. -/// -/// A blank/whitespace filter is a no-op. Returns immediately when `filter_path` -/// is `None`, so the non-filtered fast path pays nothing. -fn retain_by_filter_path(items: &mut Vec, filter_path: Option<&str>) { - let Some(raw) = filter_path else { return }; - if raw.trim().is_empty() { - return; - } - let normalized = crate::cache::normalize_filter_path(raw); - if normalized.is_empty() { - return; - } - items.retain(|it| crate::cache::path_matches_filter(&it.path, &normalized, "")); -} - -/// True when `filter_path` carries a meaningful prefix (non-blank, non-empty -/// after normalization) — the single predicate the federated search paths use -/// to decide whether to over-fetch and post-filter. Mirrors the no-op guards in -/// [`retain_by_filter_path`] so `has_filter` and the retain stay in lockstep. -fn is_meaningful_filter(filter_path: Option<&str>) -> bool { - filter_path - .map(|f| !f.trim().is_empty() && !crate::cache::normalize_filter_path(f).is_empty()) - .unwrap_or(false) -} - -/// Parse a federated `chunk_ref` into its `(peer, remote_alias, chunk_id)` -/// parts. -/// -/// Accepts the current project-namespaced shape `"/:"` and, -/// for backward compatibility, the legacy `":"` shape (no alias → -/// `None`, which falls back to group-scoped lookup on the peer). -/// -/// The `chunk_id` is taken after the *last* `':'` so peer/alias segments that -/// themselves contain a colon are not misparsed; the peer/alias split is on the -/// *first* `'/'`. -fn parse_federated_chunk_ref(chunk_ref: &str) -> Option<(&str, Option<&str>, u32)> { - let (left, id_str) = chunk_ref.rsplit_once(':')?; - let chunk_id: u32 = id_str.parse().ok()?; - match left.split_once('/') { - Some((peer, alias)) if !peer.is_empty() && !alias.is_empty() => { - Some((peer, Some(alias), chunk_id)) - } - _ => Some((left, None, chunk_id)), - } -} - -/// Best-effort extraction of the concatenated text content of a -/// `CallToolResult`. Resilient to rmcp's internal content enum shape. -fn extract_call_tool_text(result: &CallToolResult) -> String { - serde_json::to_value(result) - .ok() - .and_then(|v| { - v.get("content").and_then(|c| c.as_array()).map(|arr| { - arr.iter() - .filter_map(|item| item.get("text").and_then(|t| t.as_str())) - .collect::>() - .join("\n") - }) - }) - .unwrap_or_default() -} - -// ════════════════════════════════════════════════════════════════ -// REST API handlers (federation-friendly HTTP+JSON mirror of MCP tools). -// -// Expose the same logic over plain HTTP so a remote codesearch serve can be -// queried for federation WITHOUT an MCP session. Each handler constructs a -// throwaway `CodesearchService` bound to the live `ServeState`, invokes the -// existing `#[tool]` method, and returns the tool's JSON payload unwrapped -// from `CallToolResult`. Protected by serve's `require_auth_for_network` -// layer (same as /status, /mcp) — no separate auth code needed. -// ════════════════════════════════════════════════════════════════ -use axum::extract::{Path as AxumPath, Query as AxumQuery, State as AxumState}; -use axum::http::StatusCode; -use axum::response::Json as AxumJson; - -type RestResponse = AxumJson; -type RestError = (StatusCode, AxumJson); +type RestResponse = AxumJson; +type RestError = (StatusCode, AxumJson); /// Unwrap a `CallToolResult` into the JSON a federation client wants. /// -/// `CallToolResult` carries its payload as `Content::text(json_string)`. The +/// `CallToolResult` carries its payload as `ContentBlock::text(json_string)`. The /// normal case for search/find/explore/get_chunk is a single text item whose /// value parses as JSON, so we parse it back and return the structured value /// (clients get clean objects instead of a JSON-in-string). When the tool set @@ -7013,7 +2052,29 @@ pub(crate) async fn rest_get_chunk_handler( Ok(AxumJson(call_tool_result_to_json(result))) } -#[tool_handler] +impl CodesearchService { + /// Single composition point for the tool router: this file's own routes + /// plus every per-module router merged in. `#[tool_handler]` and both + /// ctors wire through here, so extracting a `#[tool]` method into its own + /// module only adds one `ToolRouter::merge` line — and the registration + /// test in `tests.rs` proves nothing was silently dropped (`#[tool]` fns + /// in an impl block the router macro does not scan are NOT registered). + fn merged_tool_router() -> ToolRouter { + let mut router = Self::tool_router(); + // The macro generates `find_impact_router` as an associated fn of + // CodesearchService (inside the find_impact.rs impl block), so the + // per-module routers all resolve through the type, not the module. + router.merge(Self::find_impact_router()); + router.merge(Self::search_router()); + router.merge(Self::find_router()); + router.merge(Self::explore_router()); + router.merge(Self::status_router()); + router.merge(Self::get_chunk_router()); + router + } +} + +#[tool_handler(router = Self::merged_tool_router())] impl ServerHandler for CodesearchService { fn get_info(&self) -> ServerInfo { let db_exists = self.db_path.exists(); @@ -7036,747 +2097,3 @@ impl ServerHandler for CodesearchService { ) } } - -// === Server Entry Point === - -/// Run the MCP server using stdio transport with file watching for live index updates. -/// -/// MCP server mode: how `codesearch mcp` connects to the index backend. -/// -/// - **Auto** — If `codesearch serve` is running, connect as an HTTP client; -/// otherwise fall back to local stdio mode. -/// - **Client** — Always connect to `codesearch serve` via HTTP; fail if not running. -/// - **Local** — Always use local DB in stdio mode (classic behavior). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum McpMode { - /// Connect to serve if available, otherwise local. - #[default] - Auto, - /// Always connect to serve; fail if unreachable. - Client, - /// Always use local DB (stdio). - Local, -} - -impl std::fmt::Display for McpMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - McpMode::Auto => write!(f, "auto"), - McpMode::Client => write!(f, "client"), - McpMode::Local => write!(f, "local"), - } - } -} - -impl std::str::FromStr for McpMode { - type Err = String; - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "auto" => Ok(McpMode::Auto), - "client" => Ok(McpMode::Client), - "local" => Ok(McpMode::Local), - other => Err(format!( - "invalid MCP mode '{}': must be 'auto', 'client', or 'local'", - other - )), - } - } -} - -/// Probe the serve health endpoint. Returns Ok(serve_url) if serve is alive. -async fn probe_serve_health(serve_url: &str) -> bool { - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_millis( - crate::constants::MCP_HEALTH_PROBE_TIMEOUT_MS, - )) - .build(); - let Ok(client) = client else { return false }; - let url = format!("{}{}", serve_url, crate::constants::HEALTH_PATH); - client.get(&url).send().await.is_ok() -} - -/// Run `codesearch mcp` as an HTTP client connecting to a running serve instance. -/// -/// Uses rmcp's `StreamableHttpClientWorker` with `reqwest::Client` to speak -/// MCP Streamable HTTP to the serve hub. The MCP client (e.g. Claude Code) -/// talks JSON-RPC over stdio to us, and rmcp relays to the serve HTTP endpoint. -/// Run `codesearch mcp` as a transparent stdio↔HTTP proxy to `codesearch serve`. -/// -/// Architecture: -/// Claude Desktop ──(stdio JSON-RPC)──▶ McpProxyService ──(HTTP Streamable)──▶ codesearch serve -/// -/// Every MCP request from Claude Desktop is forwarded verbatim to the serve hub and the -/// response is returned unchanged. This allows Claude Desktop — which has no repo context -/// of its own — to reach all repos managed by `codesearch serve`. -/// -/// ## Reconnect behaviour -/// -/// When `codesearch serve` goes away (restart, crash, network blip), the proxy does NOT -/// exit. Instead it: -/// 1. Keeps the stdio connection to Claude Desktop alive -/// 2. Returns "reconnecting" errors for any incoming tool calls -/// 3. Retries the HTTP connection every 3 seconds for up to 5 minutes -/// 4. On success, hot-swaps the peer — tool calls resume immediately -/// 5. After 5 minutes of failure, exits cleanly (Claude Desktop detects the disconnect) -/// -/// ## Idle disconnect behaviour -/// -/// One HTTP MCP session held open for the lifetime of the proxy keeps a request -/// permanently registered at the remote's ingress, so a scale-to-zero host never -/// sees 0 concurrent requests and never suspends the replica. To avoid that, the -/// connection is only held while it is actually being used: -/// -/// - An idle-checker ticks every `MCP_PROXY_IDLE_CHECK_INTERVAL_SECS`. Once no -/// request has been forwarded for `CODESEARCH_MCP_PROXY_IDLE_DISCONNECT_SECS` -/// (default `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`; `0` disables and restores -/// the always-connected behaviour), it clears the peer and cancels the -/// `RunningService`, closing the transport. -/// - That is a *planned* close, not an outage: it does not open a failure window -/// and does not count against `reconnect::MAX_DURATION_SECS`. The monitor task's -/// resulting `disconnect_tx` signal is recognised (via `voluntary_disconnect`) -/// and does not trigger an eager reconnect — reconnecting immediately would -/// defeat the purpose. -/// - The next `list_tools` / `call_tool` finds an empty peer slot and signals -/// `connect_request_tx`, which reconnects on demand. Failure-path reconnects -/// are unaffected and still run on their own cadence. -async fn run_mcp_client(serve_url: &str, cancel_token: CancellationToken) -> Result<()> { - use rmcp::{transport::stdio, ServiceExt}; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - let mcp_url = format!("{}{}", serve_url, crate::constants::MCP_ENDPOINT_PATH); - tracing::info!("🔗 Connecting to codesearch serve at {}", mcp_url); - - // Channels: spawned monitor tasks notify us when their connection drops. - let (disconnect_tx, mut disconnect_rx) = tokio::sync::mpsc::channel::<()>(1); - let (stdio_close_tx, mut stdio_close_rx) = tokio::sync::mpsc::channel::<()>(1); - // Capacity 1: coalescing duplicate "connect now" requests is correct. - let (connect_request_tx, mut connect_request_rx) = tokio::sync::mpsc::channel::<()>(1); - - // Shared peer state — hot-swapped on reconnect. - let peer_state: std::sync::Arc>>> = - std::sync::Arc::new(tokio::sync::RwLock::new(None)); - - // Idle-disconnect state, shared with the proxy service. - let last_activity: Arc> = - Arc::new(Mutex::new(std::time::Instant::now())); - let in_flight: Arc = Arc::new(AtomicUsize::new(0)); - // Notified whenever an on-demand `connect_to_serve` attempt (below, in the - // `connect_request_rx` arm) comes back `Err` — lets `await_peer` stop waiting - // on a definitive refusal instead of polling out the rest of its window. - let connect_failed: Arc = Arc::new(tokio::sync::Notify::new()); - // Cancellation handle for the *current* connection. `RunningServiceCancellationToken` - // is not Clone and its `cancel()` consumes self, so it lives in an Option slot - // that the idle-checker `take()`s. - let conn_cancel: Arc< - tokio::sync::Mutex>, - > = Arc::new(tokio::sync::Mutex::new(None)); - // Set just before we cancel a connection ourselves, so the disconnect signal it - // produces is not mistaken for an outage. - let voluntary_disconnect = Arc::new(AtomicBool::new(false)); - let idle_disconnect_secs = resolve_proxy_idle_disconnect_secs(None); - if idle_disconnect_secs == 0 { - tracing::info!("idle-disconnect disabled — holding the serve connection open"); - } else { - tracing::info!( - "💤 idle-disconnect enabled: closing the serve connection after {}s without traffic (checked every {}s)", - idle_disconnect_secs, - crate::constants::MCP_PROXY_IDLE_CHECK_INTERVAL_SECS - ); - } - - // Step 1: Start stdio proxy for Claude Desktop. - // This must happen first so Claude Desktop has something to talk to, - // even before the serve connection is established. - let proxy = McpProxyService { - peer: peer_state.clone(), - disconnect_tx: disconnect_tx.clone(), - connect_request_tx: connect_request_tx.clone(), - last_activity: last_activity.clone(), - in_flight: in_flight.clone(), - connect_failed: connect_failed.clone(), - }; - let server = proxy - .serve(stdio()) - .await - .context("Failed to start proxy stdio server")?; - - // Spawn a task that watches the stdio connection (takes ownership of server). - tokio::spawn(async move { - let _ = server.waiting().await; - let _ = stdio_close_tx.send(()).await; - }); - - // Step 2: Initial connection to serve (tolerant — may not be running yet). - let mut serve_down_since: Option = None; - match connect_to_serve( - &mcp_url, - &peer_state, - disconnect_tx.clone(), - &conn_cancel, - &last_activity, - ) - .await - { - Ok(()) => { - tracing::info!("🚀 MCP proxy ready — forwarding Claude Desktop ↔ codesearch serve"); - } - Err(e) => { - serve_down_since = Some(std::time::Instant::now()); - tracing::warn!( - "codesearch serve not yet available ({}). Proxy is up, will retry every {}s.", - e, - reconnect::INTERVAL_SECS - ); - // Seed a synthetic disconnect so the main loop starts reconnecting. - let tx = disconnect_tx.clone(); - tokio::spawn(async move { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let _ = tx.send(()).await; - }); - } - } - - // Step 3: Main loop — wait for stdio close, serve disconnect, an on-demand - // connect request, an idle timeout, or cancel. - - let mut idle_ticker = tokio::time::interval(std::time::Duration::from_secs( - crate::constants::MCP_PROXY_IDLE_CHECK_INTERVAL_SECS, - )); - // The first tick of a tokio interval completes immediately; skip it so a - // freshly started proxy is not evaluated for idleness before it can be used. - idle_ticker.tick().await; - - loop { - tokio::select! { - biased; // Prefer clean shutdown paths over reconnect - - // Claude Desktop closed stdio — we're done. - _ = stdio_close_rx.recv() => { - tracing::info!("MCP proxy transport closed"); - return Ok(()); - } - - // External cancel signal (e.g. process termination). - _ = cancel_token.cancelled() => { - tracing::info!("🛑 Shutdown signal received, stopping MCP proxy..."); - return Ok(()); - } - - // A request arrived while the peer slot was empty — connect now rather - // than waiting for the failure-path cadence. Ordered before the - // disconnect branch so a pending 3s backoff cannot starve it. - _ = connect_request_rx.recv() => { - if peer_state.read().await.is_some() { - continue; // Someone else already reconnected. - } - match connect_to_serve(&mcp_url, &peer_state, disconnect_tx.clone(), &conn_cancel, &last_activity).await { - Ok(()) => { - tracing::info!("🔗 Reconnected to codesearch serve on demand"); - serve_down_since = None; - } - Err(e) => { - // Serve is genuinely unreachable (or still waking). Hand over - // to the existing failure loop, which retries on its own - // cadence and eventually gives up. - tracing::debug!("On-demand connect failed: {}", e); - note_connect_failure(&connect_failed, &disconnect_tx); - } - } - } - - // Serve disconnected — enter reconnect loop. - _ = disconnect_rx.recv() => { - // Clear peer so tool calls get "reconnecting" error. - { - let mut p = peer_state.write().await; - *p = None; - } - - // A disconnect we caused on purpose (idle-close) is not an outage: - // no failure window, no eager reconnect — the next request will ask - // for one via connect_request_tx. - if voluntary_disconnect.swap(false, Ordering::SeqCst) { - tracing::debug!( - "serve connection closed after idle — will reconnect on the next request" - ); - continue; - } - - if serve_down_since.is_none() { - serve_down_since = Some(std::time::Instant::now()); - tracing::warn!( - "codesearch serve disconnected — will attempt reconnect every {}s for up to {}s", - reconnect::INTERVAL_SECS, - reconnect::MAX_DURATION_SECS, - ); - } - - let elapsed = serve_down_since.unwrap().elapsed(); - if elapsed.as_secs() > reconnect::MAX_DURATION_SECS { - tracing::error!( - "❌ Could not reconnect to serve after {}s — giving up", - reconnect::MAX_DURATION_SECS - ); - return Ok(()); // Clean exit so Claude Desktop gets graceful EOF - } - - // Wait before retrying. - tokio::time::sleep(std::time::Duration::from_secs(reconnect::INTERVAL_SECS)).await; - - match connect_to_serve(&mcp_url, &peer_state, disconnect_tx.clone(), &conn_cancel, &last_activity).await { - Ok(()) => { - tracing::info!( - "✅ Reconnected to codesearch serve (was down for {:.0}s)", - serve_down_since.unwrap().elapsed().as_secs() - ); - serve_down_since = None; - } - Err(e) => { - tracing::debug!("Reconnect attempt failed: {}", e); - // Re-trigger ourselves: the disconnect_tx from the failed - // connect_to_serve was never used, so we send a synthetic - // disconnect to keep the loop going. - let tx = disconnect_tx.clone(); - tokio::spawn(async move { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let _ = tx.send(()).await; - }); - } - } - } - - // Idle check — close the connection so a scale-to-zero remote can suspend. - _ = idle_ticker.tick() => { - if idle_disconnect_secs == 0 { - continue; // Idle-disconnect disabled. - } - let last = match last_activity.lock() { - Ok(guard) => *guard, - Err(_) => continue, // Poisoned: never tear down on a bookkeeping error. - }; - if !is_idle(last, idle_disconnect_secs, std::time::Instant::now()) { - continue; - } - - // Take the peer slot's write lock *before* checking `in_flight`, and - // hold it through the clear below, instead of checking `in_flight` - // first and taking the write lock afterwards. - // - // Every forwarding call does `InFlightGuard::new` (increments - // `in_flight`) BEFORE `self.peer.read().await` (list_tools/call_tool - // above) — so a call that has already obtained `Some(peer)` to - // forward through has necessarily already incremented the counter, - // and a call that hasn't reached the read yet will block on it once - // we hold the write lock. Reading `in_flight` first (the previous - // version) left a gap between that read and taking the write lock in - // which such a call could still slip past, get `Some(peer)`, and have - // its transport cancelled out from under it mid-request — the - // in-flight count and the peer-slot teardown were two separate - // operations pretending to be one guard. See AGENTS.md - // "counter-then-teardown races". - let mut p = peer_state.write().await; - if p.is_none() { - continue; // Already disconnected. - } - if in_flight.load(Ordering::SeqCst) > 0 { - continue; // A request is still being forwarded over this transport. - } - - tracing::info!( - "💤 Idle for {}s — closing MCP proxy connection to codesearch serve (will reconnect on next request)", - idle_disconnect_secs - ); - // Flag first, so the disconnect signal from the dying monitor task is - // recognised as planned no matter how fast it arrives. - voluntary_disconnect.store(true, Ordering::SeqCst); - *p = None; - drop(p); - if let Some(token) = conn_cancel.lock().await.take() { - token.cancel(); - } else { - // Nothing to cancel — don't leave the flag set for a later, - // genuine disconnect to misread. - voluntary_disconnect.store(false, Ordering::SeqCst); - } - } - } - } -} - -/// Establish (or re-establish) an HTTP MCP client connection to the serve hub. -/// -/// On success, updates `peer_state` with the new peer and spawns a background task -/// that monitors the connection and sends a message on `disconnect_tx` when it drops. -/// -/// Also parks the connection's cancellation handle in `conn_cancel` (so the -/// idle-checker can close it) and stamps `last_activity`, so a connection opened -/// just before an idle tick is not immediately judged idle. -async fn connect_to_serve( - mcp_url: &str, - peer_state: &std::sync::Arc>>>, - disconnect_tx: tokio::sync::mpsc::Sender<()>, - conn_cancel: &Arc>>, - last_activity: &Arc>, -) -> Result<()> { - use rmcp::ServiceExt; - - let transport = { - use rmcp::transport::streamable_http_client::{ - StreamableHttpClientTransportConfig, StreamableHttpClientWorker, - }; - let config = - StreamableHttpClientTransportConfig::with_uri(mcp_url).reinit_on_expired_session(true); - StreamableHttpClientWorker::new(reqwest::Client::new(), config) - }; - - let http_client: rmcp::service::RunningService = - ().serve(transport).await.map_err(|e| { - anyhow::anyhow!( - "Failed to connect to codesearch serve at {}.\n\ - Error: {}\n\ - Is `codesearch serve` running?", - mcp_url, - e - ) - })?; - - // Grab the cancellation handle before the RunningService is moved into the - // monitor task below — that's the only way to close this transport later - // (idle-disconnect). Cancelling it makes the monitor's `waiting()` resolve, so - // the normal disconnect path still runs; nothing needs special-casing. - { - let mut slot = conn_cancel.lock().await; - *slot = Some(http_client.cancellation_token()); - } - - // Update the shared peer. - let peer = http_client.peer().clone(); - { - let mut p = peer_state.write().await; - *p = Some(peer); - } - - // A fresh connection counts as activity: without this, an idle tick firing - // right after a reconnect would immediately close it again. - mark_proxy_activity(last_activity); - - // Spawn a monitor task that detects when the connection drops. - tokio::spawn(async move { - let _ = http_client.waiting().await; - // Connection lost — notify main loop. - let _ = disconnect_tx.send(()).await; - }); - - Ok(()) -} - -/// # Multi-instance Support -/// -/// When another instance is already running with write access to the same database, -/// this server will automatically start in **readonly mode**: -/// - Searches work normally -/// - No file watching (index won't auto-update) -/// - No incremental refresh -/// -/// This allows multiple terminal windows to use codesearch simultaneously. -pub async fn run_mcp_server( - path: Option, - create_index: bool, - log_level: crate::logger::LogLevel, - quiet: bool, - mode: McpMode, - cancel_token: CancellationToken, -) -> Result<()> { - let serve_url = serve_url_from_env(); - - // Set FASTEMBED_CACHE_DIR early (before any embedding work) to ensure fastembed - // downloads and caches models to ~/.codesearch/models instead of creating - // .fastembed_cache in the current working directory. Do this once for all modes. - match crate::constants::get_global_models_cache_dir() { - Ok(models_dir) => { - std::env::set_var("FASTEMBED_CACHE_DIR", &models_dir); - } - Err(e) => { - tracing::warn!("Could not set FASTEMBED_CACHE_DIR: {}", e); - } - } - - match mode { - McpMode::Client => { - // Client mode: init logger using global cache dir (no local DB needed) - if let Err(e) = crate::logger::init_logger( - &crate::constants::get_global_cache_dir(), - log_level, - quiet, - ) { - tracing::warn!("Failed to initialize file logger: {}", e); - } - tracing::info!("📡 MCP mode: client — connecting to serve at {}", serve_url); - if !probe_serve_health(&serve_url).await { - return Err(anyhow::anyhow!( - "codesearch serve is not running at {}. \ - Start it with `codesearch serve` or use --mode auto/local.", - serve_url - )); - } - return run_mcp_client(&serve_url, cancel_token).await; - } - McpMode::Auto => { - // Auto mode: init logger early for probe logging - if let Err(e) = crate::logger::init_logger( - &crate::constants::get_global_cache_dir(), - log_level, - quiet, - ) { - tracing::warn!("Failed to initialize file logger: {}", e); - } - if probe_serve_health(&serve_url).await { - tracing::info!( - "📡 MCP mode: auto — serve detected at {}, connecting as client", - serve_url - ); - return run_mcp_client(&serve_url, cancel_token).await; - } - tracing::info!("📡 MCP mode: auto — no serve detected, falling back to local stdio"); - // Fall through to local mode - } - McpMode::Local => { - tracing::info!("📡 MCP mode: local — using local DB (stdio)"); - // Fall through to local mode - } - } - - // ── Local stdio mode (original behavior) ────────────────────────── - use rmcp::{transport::stdio, ServiceExt}; - - tracing::info!("🚀 Starting codesearch MCP server"); - - // Use database discovery to find the best database - let db_info = find_best_database(path.as_deref())?; - - let (project_path, db_path) = if let Some(info) = db_info { - (info.project_path, info.db_path) - } else { - // No database found - if !create_index { - return Err(anyhow::anyhow!( - "No database found in current directory, parent directories, or globally tracked repositories. \ - Run 'codesearch index' first to index the codebase, or use --create-index=true flag to automatically create it." - )); - } - - // Create minimal database structure to allow server to start immediately - let effective_path = path.as_ref().cloned().unwrap_or(std::env::current_dir()?); - - // Use git root detection to place database in the correct location - let db_root = - crate::index::find_git_root(&effective_path)?.unwrap_or_else(|| effective_path.clone()); - let db_path = db_root.join(".codesearch.db"); - - tracing::info!( - "📁 Creating minimal database structure at {}", - db_path.display() - ); - - // Create directory - std::fs::create_dir_all(&db_path)?; - - // Get model info - let model_type = ModelType::default(); - let model_short_name = model_type.short_name().to_string(); - let dimensions = model_type.dimensions(); - - // Create minimal metadata.json (atomic read-modify-write, matching format - // used by build_index). Routes through the single-source-of-truth stamp so - // model_name matches the other index paths (previously wrote the Debug - // variant name here, e.g. "AllMiniLML6V2Q", instead of the model name). - crate::vectordb::merge_metadata_atomic(&db_path, |obj| { - model_type.write_metadata_fields(obj); - obj.insert( - "indexed_at".to_string(), - serde_json::Value::String(chrono::Utc::now().to_rfc3339()), - ); - })?; - - // Create minimal file_meta.json (matching FileMetaStore format) - let file_meta = crate::cache::FileMetaStore::new(model_short_name.clone(), dimensions); - file_meta.save(&db_path)?; - - // Create FTS directory - let fts_path = db_path.join("fts"); - std::fs::create_dir_all(&fts_path)?; - - // Create LMDB file by opening VectorStore (creates minimal structure) - let _store = crate::vectordb::VectorStore::new(&db_path, dimensions)?; - - tracing::info!("✅ Minimal database created successfully"); - tracing::info!("🔄 Background indexing will begin shortly via incremental refresh"); - - (effective_path, db_path) - }; - - // Initialize file logger now that db_path is known (works for both existing and auto-created DB) - // NOTE: For MCP, tracing is NOT initialized in main.rs — this is the only init call - if let Err(e) = crate::logger::init_logger(&db_path, log_level, quiet) { - tracing::warn!("Failed to initialize file logger: {}", e); - } - - tracing::info!("📂 Project: {}", project_path.display()); - tracing::info!("💾 Database: {}", db_path.display()); - - // Read model metadata to get dimensions (fallback to 384 if missing/corrupt) - let metadata_path = db_path.join("metadata.json"); - let dimensions = if metadata_path.exists() { - match std::fs::read_to_string(&metadata_path) - .ok() - .and_then(|c| serde_json::from_str::(&c).ok()) - .and_then(|j| j.get("dimensions").and_then(|v| v.as_u64())) - { - Some(d) => d as usize, - None => { - tracing::warn!( - "⚠️ Could not parse dimensions from metadata.json, using default {}", - crate::constants::DEFAULT_EMBEDDING_DIMENSIONS - ); - crate::constants::DEFAULT_EMBEDDING_DIMENSIONS - } - } - } else { - tracing::warn!( - "⚠️ metadata.json not found, using default dimensions {}", - crate::constants::DEFAULT_EMBEDDING_DIMENSIONS - ); - crate::constants::DEFAULT_EMBEDDING_DIMENSIONS - }; - - // Create shared stores - try write mode first, fall back to readonly if locked - // This enables multiple terminal windows to use the same database - tracing::info!("📦 Creating shared stores..."); - let (shared_stores, is_readonly) = SharedStores::new_or_readonly(&db_path, dimensions)?; - let shared_stores = Arc::new(shared_stores); - - if is_readonly { - tracing::warn!("🔒 Running in READONLY mode (another instance has write access)"); - tracing::warn!(" ↳ Searches work normally, but index won't auto-update"); - tracing::warn!(" ↳ Close the other instance to enable write mode"); - } - - // Create MCP service with shared stores (ready immediately) - let service = CodesearchService::new_with_stores( - Some(project_path.clone()), - Some(shared_stores.clone()), - )?; - - tracing::info!("🧠 Model: {}", service.model_type.name()); - - // START MCP SERVER NOW - fixes timeout! - tracing::info!( - "🚀 Starting MCP server{}...", - if is_readonly { " (readonly)" } else { "" } - ); - let server = service.serve(stdio()).await?; - - tracing::info!("MCP server ready. Waiting for requests..."); - - // Only run background tasks if we have write access - if !is_readonly { - // Create IndexManager with shared stores (skip initial refresh - do in background) - tracing::info!("🔍 Initializing index manager..."); - let index_manager = - IndexManager::new_without_refresh(&project_path, shared_stores.clone()).await?; - - // Background: refresh FIRST, then file watcher (sequential, not concurrent) - // Both write to SharedStores, so they must not run concurrently - let project_path_clone = project_path.clone(); - let db_path_clone = db_path.clone(); - let shared_stores_clone = shared_stores.clone(); - let index_manager_arc = Arc::new(index_manager); - let bg_cancel_token = cancel_token.clone(); - tokio::spawn(async move { - // Step 0: Pre-start FSW to collect file change events during refresh - // This ensures changes made while the refresh is running are not missed - if let Err(e) = index_manager_arc.start_watching().await { - tracing::warn!("⚠️ Could not pre-start file watcher: {}", e); - } - - // Step 1: Run initial refresh (writes to stores) - tracing::info!("🔄 Starting background incremental refresh..."); - match IndexManager::perform_incremental_refresh_with_stores( - &project_path_clone, - &db_path_clone, - &shared_stores_clone, - &bg_cancel_token, - ) - .await - { - Ok(_) => { - tracing::info!("✅ Background incremental refresh completed"); - - // Check if shutdown was requested during refresh - if bg_cancel_token.is_cancelled() { - tracing::info!("🛑 Shutdown requested, skipping file watcher startup"); - return; - } - - // Step 2: AFTER refresh completes, start file watcher (also writes to stores) - tracing::info!("👀 Starting file watcher..."); - if let Err(e) = index_manager_arc - .start_file_watcher(bg_cancel_token, None, None) - .await - { - tracing::error!("❌ Failed to start file watcher: {}", e); - } else { - tracing::info!( - "✅ File watcher active - index will auto-update on file changes" - ); - } - } - Err(e) => { - tracing::error!("❌ Background incremental refresh failed: {}", e); - } - } - }); - - // Start periodic log cleanup task - let db_path_for_cleanup = db_path.clone(); - let cleanup_cancel_token = cancel_token.clone(); - tokio::spawn(async move { - use crate::logger::{cleanup_old_logs, LogRotationConfig}; - - // Run initial cleanup on startup - let rotation_config = LogRotationConfig::from_env(); - tracing::info!("🧹 Running initial log cleanup..."); - if let Err(e) = cleanup_old_logs(&db_path_for_cleanup, &rotation_config) { - tracing::warn!("Initial log cleanup failed: {}", e); - } - - // Start periodic cleanup task (every 24 hours by default) - crate::logger::start_cleanup_task( - db_path_for_cleanup.clone(), - rotation_config, - cleanup_cancel_token, - ); - }); - } else { - tracing::info!("📖 Readonly mode: skipping background refresh and file watcher"); - } - - // Wait for shutdown: either MCP transport closes or cancellation token fires - tokio::select! { - result = server.waiting() => { - tracing::info!("MCP server transport closed"); - result?; - } - _ = cancel_token.cancelled() => { - tracing::info!("🛑 Shutdown signal received, stopping MCP server..."); - } - } - - tracing::info!("✅ MCP server shut down cleanly"); - Ok(()) -} - -#[cfg(test)] -#[path = "federation_helpers_tests.rs"] -mod federation_helpers_tests; diff --git a/src/mcp/proxy.rs b/src/mcp/proxy.rs new file mode 100644 index 00000000..3a333d4b --- /dev/null +++ b/src/mcp/proxy.rs @@ -0,0 +1,498 @@ +use rmcp::{ + model::{ + CallToolRequestParams, CallToolResponse, Implementation, ListToolsResult, + PaginatedRequestParams, ServerCapabilities, ServerInfo, + }, + service::RequestContext, + ErrorData as McpError, RoleClient, RoleServer, ServerHandler, +}; +use std::sync::{Arc, Mutex}; + +// ═══════════════════════════════════════════════════════════════════ +// MCP Proxy Service (--mode client / --mode auto with serve detected) +// ═══════════════════════════════════════════════════════════════════ + +/// Transparent stdio↔HTTP proxy with automatic reconnect. +/// +/// When `codesearch mcp --mode client` is started by Claude Desktop: +/// - Claude Desktop sends MCP requests over stdio +/// - `McpProxyService` forwards every request to the running `codesearch serve` hub via HTTP +/// - Responses flow back unchanged +/// +/// This is the correct architecture for Claude Desktop: it has no repo context of its own +/// and therefore cannot use `--mode local`. With `--mode client` it always connects to +/// the serve hub, gaining access to all registered repos. +/// +/// Only tool operations (`list_tools`, `call_tool`) are forwarded. Prompts, resources, +/// and completion are not proxied — the serve hub does not expose them. +/// +/// ## Reconnect +/// +/// The peer is wrapped in `Arc>>` so it can be hot-swapped when the +/// serve connection drops and reconnects. During reconnection, tool calls return a +/// descriptive "reconnecting" error so Claude Desktop can retry. +/// +/// ## Idle disconnect / connect on demand +/// +/// The peer is also `None` while the proxy is *deliberately* disconnected: after +/// `CODESEARCH_MCP_PROXY_IDLE_DISCONNECT_SECS` (default +/// `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`, `0` disables) without a successful +/// forwarded request, the idle-checker in `run_mcp_client` closes the HTTP MCP +/// session so a scale-to-zero remote can suspend its replica. Every successful +/// `list_tools` / `call_tool` stamps `last_activity`, which resets that window. +/// +/// Because a closed session is indistinguishable from a dead one at the peer +/// slot, `call_tool` / `list_tools` signal `connect_request_tx` on their first +/// attempt whenever the slot is `None`, asking the main loop to connect *now* +/// instead of waiting for the failure-path reconnect cadence. The existing +/// bounded retry-with-backoff remains the fallback if that connect does not land +/// within the retry budget. +pub(crate) struct McpProxyService { + /// Shared peer handle — hot-swapped on reconnect. + /// `None` means we're reconnecting to serve; tool calls return a retry-able error. + pub(crate) peer: std::sync::Arc>>>, + /// Signal to the main loop in `run_mcp_client` that the current peer is dead + /// and a fresh `connect_to_serve` should be attempted. Sent from `call_tool` / + /// `list_tools` when rmcp returns a transport-level error so we can recover + /// from server restarts and TCP keep-alive failures without bubbling the error + /// up to Claude Desktop. + pub(crate) disconnect_tx: tokio::sync::mpsc::Sender<()>, + /// Ask the main loop to run `connect_to_serve` immediately (capacity-1 + /// channel — duplicate requests coalesce, "connect now" is idempotent). + /// Sent when a request arrives while the peer slot is empty. + pub(crate) connect_request_tx: tokio::sync::mpsc::Sender<()>, + /// When the last request was successfully forwarded to serve. Shared with the + /// idle-checker in `run_mcp_client`, which closes the connection once this is + /// older than the configured idle-disconnect window. + pub(crate) last_activity: Arc>, + /// Number of requests currently being forwarded. `last_activity` only advances + /// on completion, so without this a request that runs longer than the idle + /// window (a big search, a cold symbol rebuild) would have its own transport + /// closed underneath it. The idle-checker never disconnects while this is > 0. + pub(crate) in_flight: Arc, + /// Notified by the main loop's `connect_request_rx` arm whenever an on-demand + /// `connect_to_serve` attempt returns `Err` — i.e. serve refused the + /// connection outright, as opposed to still being slow to accept one. Lets + /// `await_peer` stop waiting immediately on a definitive failure instead of + /// polling out the rest of `PROXY_CONNECT_WAIT_MS` (previously ~20s per call + /// even when serve was known to be down within the first few milliseconds). + /// A slow-but-eventually-successful wake never touches this: it resolves by + /// the peer slot filling in, which `await_peer`'s own poll already catches. + pub(crate) connect_failed: Arc, +} + +/// Keeps `McpProxyService::in_flight` incremented for its lifetime. A guard rather +/// than paired add/sub calls because the forwarding loop has several early returns. +struct InFlightGuard(Arc); + +impl InFlightGuard { + fn new(counter: &Arc) -> Self { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Self(counter.clone()) + } +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } +} + +impl McpProxyService { + #[allow(dead_code)] + fn new(peer: rmcp::service::Peer) -> Self { + // Direct constructor used by tests / single-shot scenarios. + // No reconnect plumbing — the dummy channels are never read. + let (tx, _rx) = tokio::sync::mpsc::channel(1); + let (connect_tx, _connect_rx) = tokio::sync::mpsc::channel(1); + Self { + peer: std::sync::Arc::new(tokio::sync::RwLock::new(Some(peer))), + disconnect_tx: tx, + connect_request_tx: connect_tx, + last_activity: Arc::new(Mutex::new(std::time::Instant::now())), + in_flight: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + connect_failed: Arc::new(tokio::sync::Notify::new()), + } + } + + /// Stamp "real traffic just flowed", resetting the idle-disconnect window. + fn mark_activity(&self) { + mark_proxy_activity(&self.last_activity); + } + + /// Best-effort nudge to the main loop: connect to serve now. A full channel + /// already means "a connect is pending", a closed one means the loop is gone; + /// both are fine to ignore — the caller's own retry/backoff covers it. + fn request_connect(&self) { + let _ = self.connect_request_tx.try_send(()); + } + + /// Wait — bounded by `PROXY_CONNECT_WAIT_MS` — for the peer slot to be filled + /// after `request_connect`. Returns true as soon as a peer is available. + /// + /// Without this, a request arriving after an idle-close would burn its whole + /// retry budget (~1s) while the on-demand connect is still waking a + /// scaled-to-zero remote, and fail with "reconnecting" every single time. + async fn await_peer(&self) -> bool { + self.await_peer_bounded(PROXY_CONNECT_WAIT_MS).await + } + + /// Core of `await_peer`, parameterized on the wait budget so it is unit + /// testable without actually waiting out `PROXY_CONNECT_WAIT_MS` (~20s). + /// Uses the production refusal-grace window; see + /// `await_peer_bounded_with_grace` for what that means and why it is its + /// own parameter. + async fn await_peer_bounded(&self, wait_ms: u64) -> bool { + self.await_peer_bounded_with_grace(wait_ms, CONNECT_REFUSAL_GRACE) + .await + } + + /// Core of `await_peer_bounded`, additionally parameterized on the + /// refusal-grace window so *that* is unit testable without waiting out + /// `reconnect::INTERVAL_SECS` (~3s) for real. + /// + /// Polls the peer slot on `PROXY_RETRY_BACKOFF_MS` cadence, but also races + /// each poll against `connect_failed` so a definitive on-demand connect + /// failure (serve refused the connection, not merely slow to accept one) + /// clamps the remaining wait down to `refusal_grace` instead of polling + /// out the rest of `wait_ms`. A slow-but-still-in-progress wake never + /// fires `connect_failed` — it is only notified from an `Err` return of + /// `connect_to_serve` — so this does not shorten the legitimate + /// scale-to-zero wake path, only the case where serve is already known to + /// have refused this attempt. + /// + /// The clamp is deliberately *not* an immediate return: a refusal only + /// means this one on-demand attempt was refused, not that serve won't + /// recover — `run_mcp_client`'s own disconnect/reconnect cycle + /// (`reconnect::INTERVAL_SECS` later) can still land within the original + /// budget, e.g. when serve is mid-restart rather than genuinely down. + /// Returning immediately turned that case — previously transparent to the + /// caller, since the pre-fix full-budget poll caught the reconnect — into + /// a visible "reconnecting" error on the very first request after a + /// restart. Clamping to `refusal_grace` keeps most of the original fix's + /// win (a hard-down serve is still bounded well under the full `wait_ms`) + /// while still giving that recovery cycle room to land. + async fn await_peer_bounded_with_grace( + &self, + wait_ms: u64, + refusal_grace: std::time::Duration, + ) -> bool { + let mut deadline = std::time::Instant::now() + std::time::Duration::from_millis(wait_ms); + loop { + // Register for the next failure notification BEFORE checking the + // peer slot, so a failure landing between this check and the + // `select!` below cannot be missed (the standard tokio::sync::Notify + // idiom: create the `Notified` future first, await it second). + let failed = self.connect_failed.notified(); + if self.peer.read().await.is_some() { + return true; + } + let now = std::time::Instant::now(); + if now >= deadline { + return false; + } + let backoff = std::time::Duration::from_millis(PROXY_RETRY_BACKOFF_MS) + .min(deadline.saturating_duration_since(now)); + tokio::select! { + _ = tokio::time::sleep(backoff) => {} + _ = failed => { + // A concurrent successful connect could still have landed in + // the instant before this notification; one last check keeps + // that case correct instead of reporting a false failure. + if self.peer.read().await.is_some() { + return true; + } + deadline = deadline.min(now + refusal_grace); + } + } + } + } + + /// On an empty peer slot (a deliberate idle-close or a real outage), ask the + /// main loop to connect *now* instead of waiting for the failure-path + /// reconnect cadence to notice, then give that connect a bounded window to + /// land. Returns true if a peer became available and the caller should + /// retry its forwarded call immediately. + /// + /// Only meaningful on the caller's first attempt (`attempt == 0`) — a + /// second empty slot means the on-demand connect already ran and fell + /// through to the ordinary retry/backoff path. Pulled out of `list_tools`/ + /// `call_tool` because the two copies had already started to drift (see + /// review remarks on the commit that added this). + async fn try_on_demand_connect(&self) -> bool { + self.request_connect(); + self.await_peer().await + } + + /// Force a reconnect: clear the shared peer and signal the main loop in + /// `run_mcp_client` to call `connect_to_serve` again. Brief sleep gives + /// the main loop time to actually reconnect before the caller retries. + async fn force_reconnect(&self) { + *self.peer.write().await = None; + let _ = self.disconnect_tx.send(()).await; + tokio::time::sleep(std::time::Duration::from_millis(PROXY_RETRY_BACKOFF_MS)).await; + } +} + +/// Maximum number of attempts when forwarding a request to serve. +/// Each retry includes a forced reconnect, so this also bounds reconnect attempts +/// per individual tool call. +const PROXY_MAX_RETRY_ATTEMPTS: u32 = 3; + +/// Backoff between proxy retries, also used as the post-reconnect settle delay. +const PROXY_RETRY_BACKOFF_MS: u64 = 500; + +/// How long a request may wait for an on-demand connect (after an idle-close, or +/// while serve is still starting) before falling back to the retry/backoff path. +/// +/// Sized for a scale-to-zero host: the remote's ingress *holds* the request while +/// it activates a suspended replica, so the connect itself can legitimately take +/// several seconds. Waiting here is strictly better than returning "reconnecting" +/// on the first call after every idle period. +const PROXY_CONNECT_WAIT_MS: u64 = 20_000; + +/// How long `await_peer_bounded` still waits after a definitive on-demand +/// connect refusal, instead of returning immediately or polling out the rest +/// of `PROXY_CONNECT_WAIT_MS`. +/// +/// Sized to cover `run_mcp_client`'s own disconnect/reconnect cycle +/// (`reconnect::INTERVAL_SECS`, ~3s) plus margin for the ~100ms synthetic- +/// disconnect delay and `connect_to_serve`'s own latency — so a serve that is +/// merely mid-restart still recovers transparently within this window, +/// exactly as it did before the refusal short-circuit existed, while a +/// genuinely-down serve is still bounded well under the full ~20s budget. +const CONNECT_REFUSAL_GRACE: std::time::Duration = + std::time::Duration::from_millis(reconnect::INTERVAL_SECS * 1_000 + 1_000); + +/// Record a definitive on-demand connect refusal: wake any `await_peer_bounded` +/// callers immediately (via `connect_failed`) instead of leaving them to poll +/// out their full budget for a refusal that is already known, then seed a +/// synthetic disconnect so `run_mcp_client`'s own disconnect/reconnect cycle +/// picks it up. A genuinely slow wake never reaches this function — it +/// resolves via the `Ok` branch in the caller once the peer slot fills in — +/// so this does not shorten a legitimate scale-to-zero wake, only a refusal. +/// +/// Pulled out of `run_mcp_client`'s `connect_request_rx` arm so the one line +/// that makes `await_peer_bounded`'s refusal short-circuit real in production +/// is covered by a test that calls this function directly, not only by tests +/// that call `connect_failed.notify_waiters()` themselves in isolation — +/// those pin how `await_peer_bounded` *reacts* to a notification, but nothing +/// previously pinned that this call site still *fires* one: deleting this +/// function's body left the full suite green. +pub(crate) fn note_connect_failure( + connect_failed: &tokio::sync::Notify, + disconnect_tx: &tokio::sync::mpsc::Sender<()>, +) { + connect_failed.notify_waiters(); + let tx = disconnect_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let _ = tx.send(()).await; + }); +} + +/// Heuristic: does this error message describe a transport-level failure +/// (broken TCP, server gone, stale keep-alive, stale session) that warrants +/// a forced reconnect + retry, as opposed to a real tool-level error that +/// the caller should see? +fn is_transport_error_msg(msg: &str) -> bool { + msg.contains("Transport send error") + || msg.contains("error sending request") + || msg.contains("Transport error") + || msg.contains("connection closed") + || msg.contains("error decoding response body") + || msg.contains("Session not found") + || msg.contains("404") +} + +/// Reconnect-related constants for the MCP proxy. +pub(crate) mod reconnect { + /// How long to wait between reconnect attempts. + pub const INTERVAL_SECS: u64 = 3; + /// Maximum total time to spend trying to reconnect before giving up. + pub const MAX_DURATION_SECS: u64 = 300; // 5 minutes +} + +/// Record the current instant as the proxy's most recent activity. +pub(crate) fn mark_proxy_activity(last_activity: &Arc>) { + if let Ok(mut slot) = last_activity.lock() { + *slot = std::time::Instant::now(); + } +} + +/// Resolve the MCP proxy idle-disconnect window: explicit value → env var → +/// `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`. Mirrors how `run_serve` resolves its +/// own `idle_suspend_secs`. `0` means "never idle-disconnect". +pub(crate) fn resolve_proxy_idle_disconnect_secs(explicit: Option) -> u64 { + explicit + .or_else(|| { + std::env::var(crate::constants::MCP_PROXY_IDLE_DISCONNECT_SECS_ENV) + .ok() + .and_then(|s| s.trim().parse().ok()) + }) + .unwrap_or(crate::constants::DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS) +} + +/// Has the proxy been idle long enough to close its connection to serve? +/// +/// `threshold_secs == 0` disables idle-disconnect, so this always returns false. +/// `now` is a parameter (rather than read from the clock) purely so this is unit +/// testable without sleeping. +pub(crate) fn is_idle( + last_activity: std::time::Instant, + threshold_secs: u64, + now: std::time::Instant, +) -> bool { + if threshold_secs == 0 { + return false; + } + now.saturating_duration_since(last_activity).as_secs() >= threshold_secs +} + +impl ServerHandler for McpProxyService { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info( + Implementation::new("codesearch", env!("CARGO_PKG_VERSION")) + .with_title("codesearch (serve proxy)"), + ) + .with_instructions( + "Proxy to a running codesearch serve hub. All tool calls are forwarded to the hub.", + ) + } + + async fn list_tools( + &self, + request: Option, + _cx: RequestContext, + ) -> Result { + let _in_flight = InFlightGuard::new(&self.in_flight); + let mut last_err: Option = None; + for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { + let peer = self.peer.read().await.clone(); + match peer { + Some(p) => match p.list_tools(request.clone()).await { + Ok(r) => { + self.mark_activity(); + return Ok(r); + } + Err(e) => { + let msg = e.to_string(); + if !is_transport_error_msg(&msg) || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 + { + return Err(McpError::internal_error(msg, None)); + } + tracing::warn!( + "list_tools attempt {}/{} failed (transport): {} — forcing reconnect", + attempt + 1, + PROXY_MAX_RETRY_ATTEMPTS, + msg + ); + last_err = Some(msg); + self.force_reconnect().await; + } + }, + None => { + // Empty peer slot: either a deliberate idle-close or a real + // outage. `try_on_demand_connect` asks the main loop to + // connect *now* rather than waiting for the failure-path + // reconnect cadence to notice, bounded so we still fall back + // to the ordinary retry/backoff below if it doesn't land. + if attempt == 0 && self.try_on_demand_connect().await { + continue; + } + if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { + tokio::time::sleep(std::time::Duration::from_millis( + PROXY_RETRY_BACKOFF_MS, + )) + .await; + continue; + } + return Err(McpError::internal_error( + "codesearch serve is reconnecting — please retry in a moment".to_string(), + None, + )); + } + } + } + Err(McpError::internal_error( + last_err.unwrap_or_else(|| "transport error after retries".to_string()), + None, + )) + } + + async fn call_tool( + &self, + request: CallToolRequestParams, + _cx: RequestContext, + ) -> Result { + let _in_flight = InFlightGuard::new(&self.in_flight); + let mut last_err: Option = None; + for attempt in 0..PROXY_MAX_RETRY_ATTEMPTS { + let peer = self.peer.read().await.clone(); + match peer { + Some(p) => match p.call_tool(request.clone()).await { + Ok(r) => { + self.mark_activity(); + return Ok(r.into()); + } + Err(e) => { + let msg = e.to_string(); + if !is_transport_error_msg(&msg) || attempt >= PROXY_MAX_RETRY_ATTEMPTS - 1 + { + return Err(McpError::internal_error(msg, None)); + } + tracing::warn!( + "call_tool('{}') attempt {}/{} failed (transport): {} — forcing reconnect", + request.name, + attempt + 1, + PROXY_MAX_RETRY_ATTEMPTS, + msg + ); + last_err = Some(msg); + self.force_reconnect().await; + } + }, + None => { + // Empty peer slot: either a deliberate idle-close or a real + // outage. `try_on_demand_connect` asks the main loop to + // connect *now* rather than waiting for the failure-path + // reconnect cadence to notice, bounded so we still fall back + // to the ordinary retry/backoff below if it doesn't land. + if attempt == 0 && self.try_on_demand_connect().await { + continue; + } + if attempt < PROXY_MAX_RETRY_ATTEMPTS - 1 { + tokio::time::sleep(std::time::Duration::from_millis( + PROXY_RETRY_BACKOFF_MS, + )) + .await; + continue; + } + return Err(McpError::internal_error( + "codesearch serve is reconnecting — please retry in a moment".to_string(), + None, + )); + } + } + } + Err(McpError::internal_error( + last_err.unwrap_or_else(|| "transport error after retries".to_string()), + None, + )) + } +} + +#[cfg(test)] +#[path = "proxy_idle_tests.rs"] +mod proxy_idle_tests; + +/// Unit tests for `await_peer_bounded`'s refusal clamp and the +/// `note_connect_failure` call site that fires it in production, isolated +/// from the full `run_mcp_client` loop by parameterizing the wait budget (and, +/// for the clamp itself, the refusal-grace window) so these run in +/// milliseconds instead of the real `PROXY_CONNECT_WAIT_MS` (~20s) or +/// `reconnect::INTERVAL_SECS` (~3s). +#[cfg(test)] +#[path = "await_peer_tests.rs"] +mod await_peer_tests; diff --git a/src/mcp/responses.rs b/src/mcp/responses.rs new file mode 100644 index 00000000..4507fef3 --- /dev/null +++ b/src/mcp/responses.rs @@ -0,0 +1,613 @@ +use super::helpers::prefix_path_with_alias; +use crate::embed::ModelType; +use crate::index::SharedStores; +use crate::vectordb::VectorStore; +use rmcp::model::{CallToolResult, ContentBlock}; +use rmcp::ErrorData as McpError; +use std::path::Path; +use std::sync::Arc; + +/// Read model short-name and dimensions from a database's `metadata.json`. +/// Returns `(model_name, dimensions)`, defaulting to `("unknown", DEFAULT_EMBEDDING_DIMENSIONS)`. +pub(crate) fn read_model_metadata(db_path: &Path) -> (String, usize) { + let metadata_path = db_path.join("metadata.json"); + if let Ok(content) = std::fs::read_to_string(&metadata_path) { + if let Ok(json) = serde_json::from_str::(&content) { + let model_name = json + .get("model_short_name") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + let dims = json.get("dimensions").and_then(|v| v.as_u64()).unwrap_or(0) as usize; + // If metadata has explicit dimensions, use those; otherwise infer from model name. + let dims = if dims > 0 { + dims + } else { + ModelType::parse(&model_name) + .map(|m| m.dimensions()) + .unwrap_or(crate::constants::DEFAULT_EMBEDDING_DIMENSIONS) + }; + return (model_name, dims); + } + } + ( + "unknown".to_string(), + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS, + ) +} + +/// Read chunk/file counts from metadata.json (written after each indexing operation). +/// Returns `(total_chunks, total_files)` defaulting to `(0, 0)`. +/// +/// When metadata.json reports `total_chunks == 0` but the LMDB database exists, +/// falls back to opening the store read-only and counting live chunks. +/// This catches the case where a metadata writer clobbered the stats fields +/// (see `merge_metadata_atomic` for the definitive fix). The fallback is lazy — +/// only triggered when metadata reports zero — so it does not unnecessarily +/// open databases for repos that already have correct metadata. +pub(crate) fn read_metadata_stats(db_path: &Path) -> (usize, usize) { + let metadata_path = db_path.join("metadata.json"); + if let Ok(content) = std::fs::read_to_string(&metadata_path) { + if let Ok(json) = serde_json::from_str::(&content) { + let total_chunks = json + .get("total_chunks") + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + let total_files = json + .get("total_files") + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + + if total_chunks > 0 { + return (total_chunks, total_files); + } + + // Metadata says 0 chunks — try live LMDB count as fallback. + // This is safe in serve context: `read_metadata_stats` is only called + // for repos NOT yet opened in SharedStores (opened repos use vs.stats() + // directly), so no double-open risk. + if let Some((live_chunks, live_files)) = live_chunk_count(db_path) { + tracing::info!( + "metadata.json reports 0 chunks for {}, but LMDB has {} chunks / {} files — using live count", + db_path.display(), live_chunks, live_files + ); + return (live_chunks, live_files); + } + + return (total_chunks, total_files); + } + } + (0, 0) +} + +/// Open the LMDB read-only and count chunks/files. +/// Returns `None` if the database cannot be opened (missing, corrupt, or +/// already locked by another handle). +/// +/// # Safety (LMDB double-open) +/// +/// This function is only called when `get_opened_stores(alias)` returned `None`, +/// meaning no `SharedStores` handle exists for this repo. There is a theoretical +/// race window between that check and this `open_readonly` call where another task +/// could open the repo via `get_or_open_stores`. In practice this is safe because: +/// 1. The tokio runtime uses a single thread for non-spawned futures. +/// 2. Even if the race occurs, `open_readonly` returns `Err` (TrackedEnv blocks it), +/// and we return `None` — no crash, no corruption. +pub(crate) fn live_chunk_count(db_path: &Path) -> Option<(usize, usize)> { + let (model_name, dims) = read_model_metadata(db_path); + if model_name == "unknown" { + return None; + } + match VectorStore::open_readonly(db_path, dims) { + Ok(store) => match store.stats() { + Ok(stats) if stats.total_chunks > 0 => Some((stats.total_chunks, stats.total_files)), + Ok(_) => None, + Err(e) => { + tracing::debug!( + "live_chunk_count: stats() failed for {}: {}", + db_path.display(), + e + ); + None + } + }, + Err(e) => { + tracing::debug!( + "live_chunk_count: open_readonly failed for {}: {}", + db_path.display(), + e + ); + None + } + } +} + +/// RRF score threshold below which results are considered low-confidence. +/// When the top result's RRF score falls below this, the response includes +/// a `low_confidence` flag and a `suggested_tool` hint. +pub(crate) const LOW_CONFIDENCE_THRESHOLD: f32 = 0.02; + +/// Chunk kinds that represent symbol definitions (not usages/comments/etc.) +pub(crate) const DEFINITION_KINDS: &[&str] = &[ + "Function", + "Class", + "Method", + "Struct", + "Trait", + "Enum", + "TypeAlias", + "Interface", +]; + +// === Multi-Store Routing Context === + +/// Pre-computed routing context for a tool handler. +/// +/// Created by `CodesearchService::resolve_routing()`, this struct encapsulates +/// all the decisions a handler needs: which store to use, whether to fan out, +/// and whether to call `ensure_database_exists()`. +/// Outcome of a fan-out read across several repos. +/// +/// Exists so an empty `results` is never ambiguous. A group query that hits a +/// broken store used to come back as a successful search with zero hits, which +/// is the most misleading signal this system can emit — it reads as "the corpus +/// does not contain that", and it is what sent an earlier round of this +/// investigation chasing an indexing problem that did not exist. +#[must_use] +pub(crate) struct MultiReadOutcome { + /// Merged, deduplicated, score-sorted results from the stores that worked. + pub(crate) results: Vec, + /// `(alias, full error chain)` for every store that failed. Empty on a + /// clean run. + pub(crate) failures: Vec<(String, String)>, +} + +/// Decide the `status`/`status_message` pair for a multi-store +/// `status(kind="index"|"projects")` response. +/// +/// Pulled out of the handler so the four-way call — every store down, still +/// building, ready but one or more stores failed to report their stats, or +/// fully ready — is testable without opening a single store. `failed_count` +/// is checked before declaring "building" or "ready" precisely so a store +/// that came back `Err` cannot render identically to one that returned +/// healthy zero-valued stats. The all-failed case is checked first: a +/// correlated failure (e.g. every store hits the same read-only-snapshot or +/// disk-full condition at once) also has `total_chunks == 0`, and without +/// this ordering it fell through to "building" — byte-identical to a group +/// that simply has not been indexed yet, which is the exact indistinguishable +/// case this fix exists to close. See AGENTS.md's fan-out warnings-channel +/// rule. +pub(crate) fn index_status_summary( + total_repos: usize, + failed_count: usize, + total_chunks: usize, + all_indexed: bool, +) -> (String, String) { + if total_repos > 0 && failed_count >= total_repos { + ( + "error".to_string(), + format!( + "All {total_repos} repo(s) failed to report status — every store errored, see `warnings`." + ), + ) + } else if total_chunks == 0 { + ( + "building".to_string(), + format!( + "Index is being built across {total_repos} repo(s). Searches may fail until indexing completes." + ), + ) + } else if !all_indexed { + // Chunks are in the stores but the HNSW vector index is not built yet + // (`VectorStore::search` refuses without it). Reporting "ready" here — + // the old behaviour, which keyed only off `total_chunks` — told an + // operator a rebuild was finished while every search still failed. + ( + "building".to_string(), + format!( + "Chunks are indexed across {total_repos} repo(s) but the vector index is not built yet — searches may fail until indexing completes." + ), + ) + } else if failed_count > 0 { + ( + "ready".to_string(), + format!( + "Index is ready for searching across {} of {total_repos} repo(s) — {failed_count} store(s) failed to report status, see `warnings`.", + total_repos.saturating_sub(failed_count), + ), + ) + } else { + ( + "ready".to_string(), + format!("Index is ready for searching across {total_repos} repo(s)."), + ) + } +} + +/// Status/message for a single routed store's index. +/// +/// `indexed` (the HNSW graph is built and committed) is load-bearing: a store +/// with chunks but no built graph is NOT searchable — `VectorStore::search` +/// fails with "Index not built" — so it must not read as `ready`. During a +/// rebuild there is a window where chunks are inserted but `build_index()` has +/// not run yet, which is exactly when an operator asks "is the migration done?". +pub(crate) fn single_index_status(total_chunks: usize, indexed: bool) -> (String, String) { + if total_chunks == 0 { + ( + "building".to_string(), + "Index is being built in the background. Searches may fail until indexing completes. Please check back in a few minutes.".to_string(), + ) + } else if !indexed { + ( + "building".to_string(), + "Chunks are indexed but the vector index is not built yet — searches may fail until indexing completes. Please check back in a few minutes.".to_string(), + ) + } else { + ( + "ready".to_string(), + "Index is ready for searching.".to_string(), + ) + } +} + +/// Turn a store's `stats()` result into the `(total_chunks, total_files, +/// error)` triple `list_projects` reports per repo. +/// +/// Pulled out of the handler, mirroring `index_status_summary` just above, so +/// the fix's actual claim — a `stats()` failure surfaces as `error: Some(..)` +/// with zero-valued counts, instead of silently rendering as a healthy-looking +/// empty repo — is unit-testable without opening a real `VectorStore` or +/// `ServeState`. The two calls to `serve_state.repo_lock_status()` in +/// `list_projects` don't vary by outcome, so they stay in the handler; this +/// covers only the part that does. +pub(crate) fn repo_stats_from_result( + stats: anyhow::Result, +) -> (usize, usize, Option) { + match stats { + Ok(s) => (s.total_chunks, s.total_files, None), + Err(ref e) => (0, 0, Some(format!("stats unavailable: {e:#}"))), + } +} + +/// `repo_stats_from_result` plus recording the failure as a caller-facing +/// warning, in one call. +/// +/// `list_projects` used to inline `repo_stats_from_result` and then decide +/// separately whether to push a warning — two steps a future edit could +/// silently pull apart (drop the second one, keep the first) without +/// affecting `total_chunks`/`total_files` at all, so nothing would look +/// wrong at the call site. Folding both into one call means a regression +/// that drops the warning has to delete this call entirely, which also +/// deletes the counts — no longer a silent edit. This is also the seam a +/// test can drive without opening a real `VectorStore`/`ServeState`: it +/// exercises the exact composition `list_projects` calls, not a +/// re-implementation of it. +pub(crate) fn record_stats_or_warn( + stats: anyhow::Result, + alias: &str, + warnings: &mut Vec, +) -> (usize, usize, Option) { + let (total_chunks, total_files, error) = repo_stats_from_result(stats); + if let Some(ref msg) = error { + push_store_warning(warnings, &store_warning(alias, "stats", msg)); + } + (total_chunks, total_files, error) +} + +/// Record a per-store failure as a caller-facing warning, once per store. +/// +/// Resolution loops run per hit, so a single broken store would otherwise emit +/// one identical warning per result; the caller wants to know *that* the repo +/// is down, not how many times it noticed. +pub(crate) fn note_store_failure( + warnings: &mut Vec, + aliases: &[String], + idx: usize, + what: &str, + err: &anyhow::Error, +) { + let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); + push_store_warning(warnings, &store_warning(alias, what, &format!("{err:#}"))); +} + +/// The one place a per-store warning line is formatted. Two copies used to +/// exist and could drift; a caller matching on this text would then silently +/// stop matching half of them. +pub(crate) fn store_warning(alias: &str, what: &str, err: &str) -> String { + format!("repo '{alias}' {what} failed: {err}") +} + +/// Append a warning unless it is already present, logging it once. +pub(crate) fn push_store_warning(warnings: &mut Vec, msg: &str) { + if !warnings.iter().any(|w| w == msg) { + tracing::error!("MCP: {}", msg); + warnings.push(msg.to_string()); + } +} + +/// The single exit for a handler that returns a list of items plus a warnings +/// channel. +/// +/// Five handlers previously read their channel ONLY on the empty path, so a +/// partially-failed group returned a plausible-looking short list with no +/// signal at all - the same false negative as an empty result, just harder to +/// notice. Routing every exit through here means the channel is carried +/// whether the list is empty or not, and there is no per-handler discipline +/// left to forget. +/// +/// A healthy call is byte-identical to the previous behaviour (a bare JSON +/// array), so this is backward compatible. +pub(crate) fn respond_with_items( + items: &[T], + warnings: &[String], + empty_message: impl FnOnce() -> String, +) -> Result { + respond_with_items_noted(items, warnings, None, empty_message) +} + +/// `respond_with_items` with an optional agent-facing `note` key — the shared +/// exit for item-list handlers whose result carries an advisory the caller +/// should act on (e.g. `find(kind="usages")` pointing lexical hits at +/// `find_impact`). Same discipline as `respond_with_items`: one exit, the +/// warnings channel terminates on every path. +/// +/// Shape: +/// - empty items → text via `qualify_empty_result`; when a note is present it +/// is appended to the empty message (an empty lexical result is exactly +/// where the SCIP upgrade path matters most) +/// - note + warnings → `{results, note, warnings}` +/// - note only → `{results, note}` +/// - warnings only → `{results, warnings}` (identical to `respond_with_items`) +/// - healthy, no note → bare JSON array, byte-identical to the legacy shape +pub(crate) fn respond_with_items_noted( + items: &[T], + warnings: &[String], + note: Option<&str>, + empty_message: impl FnOnce() -> String, +) -> Result { + if items.is_empty() { + let mut message = empty_message(); + if let Some(note) = note { + message.push(' '); + message.push_str(note); + } + return Ok(CallToolResult::success(vec![ContentBlock::text( + qualify_empty_result(message, warnings), + )])); + } + if note.is_none() && warnings.is_empty() { + let json = serde_json::to_string(items).unwrap_or_else(|_| "[]".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + let mut payload = serde_json::Map::new(); + payload.insert("results".to_string(), serde_json::json!(items)); + if let Some(note) = note { + payload.insert("note".to_string(), serde_json::json!(note)); + } + if !warnings.is_empty() { + payload.insert("warnings".to_string(), serde_json::json!(warnings)); + } + Ok(CallToolResult::success(vec![ContentBlock::text( + serde_json::Value::Object(payload).to_string(), + )])) +} + +/// The object-shaped sibling of `respond_with_items`: one exit for handlers that +/// return a single struct rather than a list. +/// +/// A `warnings` *field* on the response struct was the obvious fix and is the +/// weaker one — the handler is still free to populate it with `None`, and a test +/// that builds the struct itself cannot see that happen. Review round 8 proved +/// it: the round-7 defect was reintroduced at the `get_chunk` success path and +/// all 630 tests still passed. +/// +/// **This is an improvement, not a guarantee.** Round 9 measured the difference: +/// passing `&[]` here is exactly as writable as `warnings: None` was, the suite +/// still cannot see it, and no lint fires (the channel stays "used" by the +/// ambiguous path). What it actually buys is narrower and real — no optional +/// field whose absence is invisible, no future construction site that can zero +/// it, and an audit that collapses from "check every response struct" to "check +/// the call sites of two functions", which is grep-answerable. The channel can +/// no longer be *forgotten*, only actively discarded. +/// +/// Healthy path serializes the struct directly, so its key order and bytes are +/// unchanged. `serde_json::Map` is a `BTreeMap` here (no `preserve_order` +/// feature), so round-tripping through `to_value` would silently re-sort the +/// keys — which is only acceptable on the warning path, where the shape is new +/// anyway. +pub(crate) fn respond_with_object( + value: &T, + warnings: &[String], +) -> Result { + if !warnings.is_empty() { + if let Ok(mut v) = serde_json::to_value(value) { + if let Some(obj) = v.as_object_mut() { + obj.insert("warnings".to_string(), serde_json::json!(warnings)); + return Ok(CallToolResult::success(vec![ContentBlock::text( + v.to_string(), + )])); + } + } + } + let json = serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) +} + +/// Build the `ambiguous_chunk_id` payload for `get_chunk`. +/// +/// `candidate_projects` reads as the complete set of repos holding this +/// chunk_id, so a store that failed to answer must be declared: the repo the +/// caller actually wants may be the one missing from the list. Extracted so the +/// "is this list complete?" decision is testable without standing up stores. +/// +/// `warnings` is *inserted* rather than emitted as `null`, so the healthy-path +/// shape is byte-identical to before — matching `skip_serializing_if` on every +/// other warnings-carrying response. +pub(crate) fn ambiguous_chunk_payload( + chunk_id: u32, + candidate_projects: &[&str], + warnings: &[String], +) -> serde_json::Value { + let mut message = + format!("chunk_id {chunk_id} exists in multiple repositories. Specify which one."); + if !warnings.is_empty() { + message.push_str(" The candidate list is incomplete — see `warnings`."); + } + let mut payload = serde_json::json!({ + "error_code": "ambiguous_chunk_id", + "message": message, + "candidate_projects": candidate_projects, + "hint_for_agent": "The chunk_id collision is a known limitation of multi-repo mode. Re-run get_chunk with one of the candidate_projects, or use search to identify the correct repository first." + }); + if !warnings.is_empty() { + if let Some(obj) = payload.as_object_mut() { + obj.insert("warnings".to_string(), serde_json::json!(warnings)); + } + } + payload +} + +/// Decide whether to keep the "try another tool" hint. +/// +/// A weak or empty result caused by a store that is DOWN is not a reason to +/// retry with a different tool — that just sends the agent back at the same +/// broken store. Extracted from `build_semantic_response` so the decision is +/// testable without standing up a service. +pub(crate) fn retry_hint( + suggested: Option, + warnings: &Option>, +) -> Option { + // `is_some()` alone is wrong: an empty `Some(vec![])` means nothing failed, + // and suppressing a legitimate hint on it would be a silent regression the + // moment a caller constructs the warnings vec eagerly. + if warnings.as_ref().is_some_and(|w| !w.is_empty()) { + return None; + } + suggested +} + +/// Qualify a "nothing found" message when a store in scope actually failed. +/// +/// This is the defect that keeps coming back in a new handler: "No definition +/// found — the symbol may not be indexed" is a *diagnosis*, and it is flatly +/// wrong when the store never answered. An agent acts on it by giving up or by +/// re-indexing something that was never broken. +pub(crate) fn qualify_empty_result(message: String, warnings: &[String]) -> String { + if warnings.is_empty() { + return message; + } + format!( + "{message}\n\nWARNING: this result is not trustworthy — {count} store(s) in \ + scope failed, so \"not found\" may mean \"not searched\":\n{detail}", + count = warnings.len(), + detail = warnings.join("\n") + ) +} + +// Hand-written rather than derived: `derive(Default)` would demand `R: Default`, +// which the result types do not implement and do not need to. +impl Default for MultiReadOutcome { + fn default() -> Self { + Self { + results: Vec::new(), + failures: Vec::new(), + } + } +} + +impl MultiReadOutcome { + /// Render failures as caller-facing warning lines. + pub(crate) fn warnings(&self, what: &str) -> Vec { + self.failures + .iter() + .map(|(alias, err)| store_warning(alias, what, err)) + .collect() + } + + /// Take the results, routing any failures into `warnings` on the way out. + /// + /// Deliberately the only ergonomic way to get at `results`: reaching for + /// the field directly and dropping `failures` is `unwrap_or_default()` + /// under a new name, and that is the bug this whole type exists to stop. + pub(crate) fn into_results(self, warnings: &mut Vec, what: &str) -> Vec { + for (alias, err) in &self.failures { + push_store_warning(warnings, &store_warning(alias, what, err)); + } + self.results + } +} + +pub(crate) struct MultiStoreContext { + /// Single-store override (set when exactly 1 repo resolved, or None). + /// Pass to `with_*_store_read_for()` methods. + pub(crate) stores: Option>, + /// Multi-store vec for fan-out (set when 2+ repos resolved, or None). + /// Use `if let Some(ref sv) = ctx.stores_vec { ... }` for the multi-store path. + pub(crate) stores_vec: Option>>, + /// Alias for each store in `stores_vec` (parallel with stores_vec). + /// Used for path prefixing and per-alias dedup. + pub(crate) store_aliases: Option>, + /// Alias for single-project routing (set when project= is given). + pub(crate) project_alias: Option, + /// Normalized project root for each alias (alias → root path). + /// Used by `prefix_path` to strip absolute paths and add alias prefix. + pub(crate) alias_roots: std::collections::HashMap, + /// True when `stores_vec` has 2+ entries (group fan-out). + pub(crate) is_multi: bool, + /// True when no serve-state stores resolved and local DB should be checked. + pub(crate) needs_local_db: bool, +} + +impl MultiStoreContext { + /// Aliases parallel to `stores_vec`, or an empty slice when absent. + /// + /// Every fan-out that reports a per-store failure needs this, and hand-rolling + /// `let empty = Vec::new(); ...unwrap_or(&empty)` at each site produced four + /// copies of the same two lines — and one handler where the binding was out + /// of scope, which is how a silent store read survived a round of review. + pub(crate) fn aliases(&self) -> &[String] { + self.store_aliases.as_deref().unwrap_or(&[]) + } + + /// Prefix a result path with its owning alias for multi-repo identification. + /// + /// Three dispatch modes: + /// - Single-project (`project_alias = Some(...)`): prefix with that alias. + /// - Group (`store_aliases = Some([...])`): detect alias by prefix-matching + /// the path against known project roots in `alias_roots`. + /// - Stdio / no alias info: normalize only, no prefix. + /// + /// Emits a `tracing::debug!` event when an expected alias cannot be resolved. + /// That usually indicates a config mismatch or a path from an unregistered source — + /// the path is still normalized and returned, but diagnosis is easier with the log. + pub(crate) fn prefix_result_path(&self, path: &str) -> String { + if let Some(ref alias) = self.project_alias { + if let Some(root) = self.alias_roots.get(alias) { + return prefix_path_with_alias(path, Some(alias), root); + } + tracing::debug!( + target: "codesearch::mcp::path_prefix", + alias = %alias, + path = %path, + "project_alias has no entry in alias_roots" + ); + } + if let Some(ref aliases) = self.store_aliases { + let normalized = crate::cache::normalize_path_str(path); + for alias in aliases { + if let Some(root) = self.alias_roots.get(alias) { + if normalized.starts_with(root.as_str()) { + return prefix_path_with_alias(path, Some(alias), root); + } + } + } + tracing::debug!( + target: "codesearch::mcp::path_prefix", + aliases = ?aliases, + path = %path, + "no alias root matched path in group mode" + ); + } + crate::cache::normalize_path_str(path) + } +} diff --git a/src/mcp/runtime.rs b/src/mcp/runtime.rs new file mode 100644 index 00000000..f44999c1 --- /dev/null +++ b/src/mcp/runtime.rs @@ -0,0 +1,899 @@ +use super::proxy::{ + is_idle, mark_proxy_activity, note_connect_failure, reconnect, + resolve_proxy_idle_disconnect_secs, McpProxyService, +}; +use super::{serve_url_from_env, CodesearchService}; +use crate::db_discovery::find_best_database; +use crate::embed::ModelType; +use crate::index::{IndexManager, SharedStores}; +use anyhow::{Context, Result}; +use rmcp::RoleClient; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use tokio_util::sync::CancellationToken; + +// === Server Entry Point === + +/// Run the MCP server using stdio transport with file watching for live index updates. +/// +/// MCP server mode: how `codesearch mcp` connects to the index backend. +/// +/// - **Auto** — If `codesearch serve` is running, connect as an HTTP client; +/// otherwise fall back to local stdio mode. +/// - **Client** — Always connect to `codesearch serve` via HTTP; fail if not running. +/// - **Local** — Always use local DB in stdio mode (classic behavior). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum McpMode { + /// Connect to serve if available, otherwise local. + #[default] + Auto, + /// Always connect to serve; fail if unreachable. + Client, + /// Always use local DB (stdio). + Local, +} + +impl std::fmt::Display for McpMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + McpMode::Auto => write!(f, "auto"), + McpMode::Client => write!(f, "client"), + McpMode::Local => write!(f, "local"), + } + } +} + +impl std::str::FromStr for McpMode { + type Err = String; + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "auto" => Ok(McpMode::Auto), + "client" => Ok(McpMode::Client), + "local" => Ok(McpMode::Local), + other => Err(format!( + "invalid MCP mode '{}': must be 'auto', 'client', or 'local'", + other + )), + } + } +} + +pub(crate) fn validate_local_startup_flags( + mode: McpMode, + readonly: bool, + require_ready: bool, +) -> Result<()> { + if mode != McpMode::Local && (readonly || require_ready) { + return Err(anyhow::anyhow!( + "--readonly and --require-ready apply only to local MCP mode; \ + pass --mode local instead of --mode {}", + mode, + )); + } + Ok(()) +} + +pub(crate) fn may_create_missing_index( + create_index: bool, + readonly: bool, + require_ready: bool, +) -> bool { + create_index && !readonly && !require_ready +} + +#[derive(Debug, Clone, Copy)] +pub struct McpStartupOptions { + pub create_index: bool, + pub readonly: bool, + pub require_ready: bool, +} + +pub(crate) fn validate_prebuilt_index_health( + db_path: &Path, + total_chunks: usize, + vector_indexed: bool, + fts_documents: usize, + partial: bool, +) -> Result<()> { + if total_chunks == 0 || !vector_indexed || fts_documents == 0 || partial { + return Err(anyhow::anyhow!( + "Prebuilt codesearch index is not ready at {}: \ + chunks={}, vector_indexed={}, fts_documents={}, partial={}. \ + Run `codesearch index` before starting MCP with --require-ready.", + db_path.display(), + total_chunks, + vector_indexed, + fts_documents, + partial, + )); + } + Ok(()) +} + +pub(crate) async fn require_prebuilt_index_ready( + stores: &SharedStores, + db_path: &Path, +) -> Result<()> { + let (total_chunks, vector_indexed) = { + let vector_store = stores.vector_store.read().await; + vector_store + .index_health() + .context("Failed to inspect prebuilt vector index")? + }; + let fts_documents = { + let fts_store = stores.fts_store.read().await; + fts_store + .stats() + .context("Failed to inspect prebuilt full-text index")? + .num_documents + }; + let metadata = std::fs::read_to_string(db_path.join("metadata.json")) + .context("Failed to read prebuilt index metadata")?; + let metadata: serde_json::Value = + serde_json::from_str(&metadata).context("Failed to parse prebuilt index metadata")?; + let partial = match metadata.get("partial") { + None => { + return Err(anyhow::anyhow!( + "Prebuilt index metadata is missing the partial readiness marker; \ + run `codesearch index` before starting MCP with --require-ready" + )); + } + Some(value) => value + .as_bool() + .ok_or_else(|| anyhow::anyhow!("Prebuilt index metadata partial must be a boolean"))?, + }; + validate_prebuilt_index_health( + db_path, + total_chunks, + vector_indexed, + fts_documents, + partial, + ) +} + +/// Probe the serve health endpoint. Returns Ok(serve_url) if serve is alive. +async fn probe_serve_health(serve_url: &str) -> bool { + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_millis( + crate::constants::MCP_HEALTH_PROBE_TIMEOUT_MS, + )) + .build(); + let Ok(client) = client else { return false }; + let url = format!("{}{}", serve_url, crate::constants::HEALTH_PATH); + client.get(&url).send().await.is_ok() +} + +/// Run `codesearch mcp` as an HTTP client connecting to a running serve instance. +/// +/// Uses rmcp's `StreamableHttpClientWorker` with `reqwest::Client` to speak +/// MCP Streamable HTTP to the serve hub. The MCP client (e.g. Claude Code) +/// talks JSON-RPC over stdio to us, and rmcp relays to the serve HTTP endpoint. +/// Run `codesearch mcp` as a transparent stdio↔HTTP proxy to `codesearch serve`. +/// +/// Architecture: +/// Claude Desktop ──(stdio JSON-RPC)──▶ McpProxyService ──(HTTP Streamable)──▶ codesearch serve +/// +/// Every MCP request from Claude Desktop is forwarded verbatim to the serve hub and the +/// response is returned unchanged. This allows Claude Desktop — which has no repo context +/// of its own — to reach all repos managed by `codesearch serve`. +/// +/// ## Reconnect behaviour +/// +/// When `codesearch serve` goes away (restart, crash, network blip), the proxy does NOT +/// exit. Instead it: +/// 1. Keeps the stdio connection to Claude Desktop alive +/// 2. Returns "reconnecting" errors for any incoming tool calls +/// 3. Retries the HTTP connection every 3 seconds for up to 5 minutes +/// 4. On success, hot-swaps the peer — tool calls resume immediately +/// 5. After 5 minutes of failure, exits cleanly (Claude Desktop detects the disconnect) +/// +/// ## Idle disconnect behaviour +/// +/// One HTTP MCP session held open for the lifetime of the proxy keeps a request +/// permanently registered at the remote's ingress, so a scale-to-zero host never +/// sees 0 concurrent requests and never suspends the replica. To avoid that, the +/// connection is only held while it is actually being used: +/// +/// - An idle-checker ticks every `MCP_PROXY_IDLE_CHECK_INTERVAL_SECS`. Once no +/// request has been forwarded for `CODESEARCH_MCP_PROXY_IDLE_DISCONNECT_SECS` +/// (default `DEFAULT_MCP_PROXY_IDLE_DISCONNECT_SECS`; `0` disables and restores +/// the always-connected behaviour), it clears the peer and cancels the +/// `RunningService`, closing the transport. +/// - That is a *planned* close, not an outage: it does not open a failure window +/// and does not count against `reconnect::MAX_DURATION_SECS`. The monitor task's +/// resulting `disconnect_tx` signal is recognised (via `voluntary_disconnect`) +/// and does not trigger an eager reconnect — reconnecting immediately would +/// defeat the purpose. +/// - The next `list_tools` / `call_tool` finds an empty peer slot and signals +/// `connect_request_tx`, which reconnects on demand. Failure-path reconnects +/// are unaffected and still run on their own cadence. +async fn run_mcp_client(serve_url: &str, cancel_token: CancellationToken) -> Result<()> { + use rmcp::{transport::stdio, ServiceExt}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + let mcp_url = format!("{}{}", serve_url, crate::constants::MCP_ENDPOINT_PATH); + tracing::info!("🔗 Connecting to codesearch serve at {}", mcp_url); + + // Channels: spawned monitor tasks notify us when their connection drops. + let (disconnect_tx, mut disconnect_rx) = tokio::sync::mpsc::channel::<()>(1); + let (stdio_close_tx, mut stdio_close_rx) = tokio::sync::mpsc::channel::<()>(1); + // Capacity 1: coalescing duplicate "connect now" requests is correct. + let (connect_request_tx, mut connect_request_rx) = tokio::sync::mpsc::channel::<()>(1); + + // Shared peer state — hot-swapped on reconnect. + let peer_state: std::sync::Arc>>> = + std::sync::Arc::new(tokio::sync::RwLock::new(None)); + + // Idle-disconnect state, shared with the proxy service. + let last_activity: Arc> = + Arc::new(Mutex::new(std::time::Instant::now())); + let in_flight: Arc = Arc::new(AtomicUsize::new(0)); + // Notified whenever an on-demand `connect_to_serve` attempt (below, in the + // `connect_request_rx` arm) comes back `Err` — lets `await_peer` stop waiting + // on a definitive refusal instead of polling out the rest of its window. + let connect_failed: Arc = Arc::new(tokio::sync::Notify::new()); + // Cancellation handle for the *current* connection. `RunningServiceCancellationToken` + // is not Clone and its `cancel()` consumes self, so it lives in an Option slot + // that the idle-checker `take()`s. + let conn_cancel: Arc< + tokio::sync::Mutex>, + > = Arc::new(tokio::sync::Mutex::new(None)); + // Set just before we cancel a connection ourselves, so the disconnect signal it + // produces is not mistaken for an outage. + let voluntary_disconnect = Arc::new(AtomicBool::new(false)); + let idle_disconnect_secs = resolve_proxy_idle_disconnect_secs(None); + if idle_disconnect_secs == 0 { + tracing::info!("idle-disconnect disabled — holding the serve connection open"); + } else { + tracing::info!( + "💤 idle-disconnect enabled: closing the serve connection after {}s without traffic (checked every {}s)", + idle_disconnect_secs, + crate::constants::MCP_PROXY_IDLE_CHECK_INTERVAL_SECS + ); + } + + // Step 1: Start stdio proxy for Claude Desktop. + // This must happen first so Claude Desktop has something to talk to, + // even before the serve connection is established. + let proxy = McpProxyService { + peer: peer_state.clone(), + disconnect_tx: disconnect_tx.clone(), + connect_request_tx: connect_request_tx.clone(), + last_activity: last_activity.clone(), + in_flight: in_flight.clone(), + connect_failed: connect_failed.clone(), + }; + let server = proxy + .serve(stdio()) + .await + .context("Failed to start proxy stdio server")?; + + // Spawn a task that watches the stdio connection (takes ownership of server). + tokio::spawn(async move { + let _ = server.waiting().await; + let _ = stdio_close_tx.send(()).await; + }); + + // Step 2: Initial connection to serve (tolerant — may not be running yet). + let mut serve_down_since: Option = None; + match connect_to_serve( + &mcp_url, + &peer_state, + disconnect_tx.clone(), + &conn_cancel, + &last_activity, + ) + .await + { + Ok(()) => { + tracing::info!("🚀 MCP proxy ready — forwarding Claude Desktop ↔ codesearch serve"); + } + Err(e) => { + serve_down_since = Some(std::time::Instant::now()); + tracing::warn!( + "codesearch serve not yet available ({}). Proxy is up, will retry every {}s.", + e, + reconnect::INTERVAL_SECS + ); + // Seed a synthetic disconnect so the main loop starts reconnecting. + let tx = disconnect_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let _ = tx.send(()).await; + }); + } + } + + // Step 3: Main loop — wait for stdio close, serve disconnect, an on-demand + // connect request, an idle timeout, or cancel. + + let mut idle_ticker = tokio::time::interval(std::time::Duration::from_secs( + crate::constants::MCP_PROXY_IDLE_CHECK_INTERVAL_SECS, + )); + // The first tick of a tokio interval completes immediately; skip it so a + // freshly started proxy is not evaluated for idleness before it can be used. + idle_ticker.tick().await; + + loop { + tokio::select! { + biased; // Prefer clean shutdown paths over reconnect + + // Claude Desktop closed stdio — we're done. + _ = stdio_close_rx.recv() => { + tracing::info!("MCP proxy transport closed"); + return Ok(()); + } + + // External cancel signal (e.g. process termination). + _ = cancel_token.cancelled() => { + tracing::info!("🛑 Shutdown signal received, stopping MCP proxy..."); + return Ok(()); + } + + // A request arrived while the peer slot was empty — connect now rather + // than waiting for the failure-path cadence. Ordered before the + // disconnect branch so a pending 3s backoff cannot starve it. + _ = connect_request_rx.recv() => { + if peer_state.read().await.is_some() { + continue; // Someone else already reconnected. + } + match connect_to_serve(&mcp_url, &peer_state, disconnect_tx.clone(), &conn_cancel, &last_activity).await { + Ok(()) => { + tracing::info!("🔗 Reconnected to codesearch serve on demand"); + serve_down_since = None; + } + Err(e) => { + // Serve is genuinely unreachable (or still waking). Hand over + // to the existing failure loop, which retries on its own + // cadence and eventually gives up. + tracing::debug!("On-demand connect failed: {}", e); + note_connect_failure(&connect_failed, &disconnect_tx); + } + } + } + + // Serve disconnected — enter reconnect loop. + _ = disconnect_rx.recv() => { + // Clear peer so tool calls get "reconnecting" error. + { + let mut p = peer_state.write().await; + *p = None; + } + + // A disconnect we caused on purpose (idle-close) is not an outage: + // no failure window, no eager reconnect — the next request will ask + // for one via connect_request_tx. + if voluntary_disconnect.swap(false, Ordering::SeqCst) { + tracing::debug!( + "serve connection closed after idle — will reconnect on the next request" + ); + continue; + } + + if serve_down_since.is_none() { + serve_down_since = Some(std::time::Instant::now()); + tracing::warn!( + "codesearch serve disconnected — will attempt reconnect every {}s for up to {}s", + reconnect::INTERVAL_SECS, + reconnect::MAX_DURATION_SECS, + ); + } + + let elapsed = serve_down_since.unwrap().elapsed(); + if elapsed.as_secs() > reconnect::MAX_DURATION_SECS { + tracing::error!( + "❌ Could not reconnect to serve after {}s — giving up", + reconnect::MAX_DURATION_SECS + ); + return Ok(()); // Clean exit so Claude Desktop gets graceful EOF + } + + // Wait before retrying. + tokio::time::sleep(std::time::Duration::from_secs(reconnect::INTERVAL_SECS)).await; + + match connect_to_serve(&mcp_url, &peer_state, disconnect_tx.clone(), &conn_cancel, &last_activity).await { + Ok(()) => { + tracing::info!( + "✅ Reconnected to codesearch serve (was down for {:.0}s)", + serve_down_since.unwrap().elapsed().as_secs() + ); + serve_down_since = None; + } + Err(e) => { + tracing::debug!("Reconnect attempt failed: {}", e); + // Re-trigger ourselves: the disconnect_tx from the failed + // connect_to_serve was never used, so we send a synthetic + // disconnect to keep the loop going. + let tx = disconnect_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let _ = tx.send(()).await; + }); + } + } + } + + // Idle check — close the connection so a scale-to-zero remote can suspend. + _ = idle_ticker.tick() => { + if idle_disconnect_secs == 0 { + continue; // Idle-disconnect disabled. + } + let last = match last_activity.lock() { + Ok(guard) => *guard, + Err(_) => continue, // Poisoned: never tear down on a bookkeeping error. + }; + if !is_idle(last, idle_disconnect_secs, std::time::Instant::now()) { + continue; + } + + // Take the peer slot's write lock *before* checking `in_flight`, and + // hold it through the clear below, instead of checking `in_flight` + // first and taking the write lock afterwards. + // + // Every forwarding call does `InFlightGuard::new` (increments + // `in_flight`) BEFORE `self.peer.read().await` (list_tools/call_tool + // above) — so a call that has already obtained `Some(peer)` to + // forward through has necessarily already incremented the counter, + // and a call that hasn't reached the read yet will block on it once + // we hold the write lock. Reading `in_flight` first (the previous + // version) left a gap between that read and taking the write lock in + // which such a call could still slip past, get `Some(peer)`, and have + // its transport cancelled out from under it mid-request — the + // in-flight count and the peer-slot teardown were two separate + // operations pretending to be one guard. See AGENTS.md + // "counter-then-teardown races". + let mut p = peer_state.write().await; + if p.is_none() { + continue; // Already disconnected. + } + if in_flight.load(Ordering::SeqCst) > 0 { + continue; // A request is still being forwarded over this transport. + } + + tracing::info!( + "💤 Idle for {}s — closing MCP proxy connection to codesearch serve (will reconnect on next request)", + idle_disconnect_secs + ); + // Flag first, so the disconnect signal from the dying monitor task is + // recognised as planned no matter how fast it arrives. + voluntary_disconnect.store(true, Ordering::SeqCst); + *p = None; + drop(p); + if let Some(token) = conn_cancel.lock().await.take() { + token.cancel(); + } else { + // Nothing to cancel — don't leave the flag set for a later, + // genuine disconnect to misread. + voluntary_disconnect.store(false, Ordering::SeqCst); + } + } + } + } +} + +/// Establish (or re-establish) an HTTP MCP client connection to the serve hub. +/// +/// On success, updates `peer_state` with the new peer and spawns a background task +/// that monitors the connection and sends a message on `disconnect_tx` when it drops. +/// +/// Also parks the connection's cancellation handle in `conn_cancel` (so the +/// idle-checker can close it) and stamps `last_activity`, so a connection opened +/// just before an idle tick is not immediately judged idle. +async fn connect_to_serve( + mcp_url: &str, + peer_state: &std::sync::Arc>>>, + disconnect_tx: tokio::sync::mpsc::Sender<()>, + conn_cancel: &Arc>>, + last_activity: &Arc>, +) -> Result<()> { + use rmcp::ServiceExt; + + let transport = { + use rmcp::transport::streamable_http_client::{ + StreamableHttpClientTransportConfig, StreamableHttpClientWorker, + }; + let config = + StreamableHttpClientTransportConfig::with_uri(mcp_url).reinit_on_expired_session(true); + StreamableHttpClientWorker::new(reqwest::Client::new(), config) + }; + + let http_client: rmcp::service::RunningService = + ().serve(transport).await.map_err(|e| { + anyhow::anyhow!( + "Failed to connect to codesearch serve at {}.\n\ + Error: {}\n\ + Is `codesearch serve` running?", + mcp_url, + e + ) + })?; + + // Grab the cancellation handle before the RunningService is moved into the + // monitor task below — that's the only way to close this transport later + // (idle-disconnect). Cancelling it makes the monitor's `waiting()` resolve, so + // the normal disconnect path still runs; nothing needs special-casing. + { + let mut slot = conn_cancel.lock().await; + *slot = Some(http_client.cancellation_token()); + } + + // Update the shared peer. + let peer = http_client.peer().clone(); + { + let mut p = peer_state.write().await; + *p = Some(peer); + } + + // A fresh connection counts as activity: without this, an idle tick firing + // right after a reconnect would immediately close it again. + mark_proxy_activity(last_activity); + + // Spawn a monitor task that detects when the connection drops. + tokio::spawn(async move { + let _ = http_client.waiting().await; + // Connection lost — notify main loop. + let _ = disconnect_tx.send(()).await; + }); + + Ok(()) +} + +/// # Multi-instance Support +/// +/// When another instance is already running with write access to the same database, +/// this server will automatically start in **readonly mode**: +/// - Searches work normally +/// - No file watching (index won't auto-update) +/// - No incremental refresh +/// +/// This allows multiple terminal windows to use codesearch simultaneously. +/// +/// This library entry retains the original writable, non-blocking startup +/// behavior. CLI callers that need explicit ownership controls use +/// [`run_mcp_server_with_options`]. +pub async fn run_mcp_server( + path: Option, + create_index: bool, + log_level: crate::logger::LogLevel, + quiet: bool, + mode: McpMode, + cancel_token: CancellationToken, +) -> Result<()> { + run_mcp_server_with_options( + path, + McpStartupOptions { + create_index, + readonly: false, + require_ready: false, + }, + log_level, + quiet, + mode, + cancel_token, + ) + .await +} + +/// Run MCP with explicit local index ownership and readiness controls. +/// +/// `startup.readonly` requests read-only mode up front. +/// `startup.require_ready` independently rejects an incomplete index at startup. +/// They are separate because a read-only secondary window over an index another +/// process is still building is legitimate. +pub async fn run_mcp_server_with_options( + path: Option, + startup: McpStartupOptions, + log_level: crate::logger::LogLevel, + quiet: bool, + mode: McpMode, + cancel_token: CancellationToken, +) -> Result<()> { + validate_local_startup_flags(mode, startup.readonly, startup.require_ready)?; + let serve_url = serve_url_from_env(); + + // Set FASTEMBED_CACHE_DIR early (before any embedding work) to ensure fastembed + // downloads and caches models to ~/.codesearch/models instead of creating + // .fastembed_cache in the current working directory. Do this once for all modes. + match crate::constants::get_global_models_cache_dir() { + Ok(models_dir) => { + std::env::set_var("FASTEMBED_CACHE_DIR", &models_dir); + } + Err(e) => { + tracing::warn!("Could not set FASTEMBED_CACHE_DIR: {}", e); + } + } + + match mode { + McpMode::Client => { + // Client mode: init logger using global cache dir (no local DB needed) + if let Err(e) = crate::logger::init_logger( + &crate::constants::get_global_cache_dir(), + log_level, + quiet, + ) { + tracing::warn!("Failed to initialize file logger: {}", e); + } + tracing::info!("📡 MCP mode: client — connecting to serve at {}", serve_url); + if !probe_serve_health(&serve_url).await { + return Err(anyhow::anyhow!( + "codesearch serve is not running at {}. \ + Start it with `codesearch serve` or use --mode auto/local.", + serve_url + )); + } + return run_mcp_client(&serve_url, cancel_token).await; + } + McpMode::Auto => { + // Auto mode: init logger early for probe logging + if let Err(e) = crate::logger::init_logger( + &crate::constants::get_global_cache_dir(), + log_level, + quiet, + ) { + tracing::warn!("Failed to initialize file logger: {}", e); + } + if probe_serve_health(&serve_url).await { + tracing::info!( + "📡 MCP mode: auto — serve detected at {}, connecting as client", + serve_url + ); + return run_mcp_client(&serve_url, cancel_token).await; + } + tracing::info!("📡 MCP mode: auto — no serve detected, falling back to local stdio"); + // Fall through to local mode + } + McpMode::Local => { + tracing::info!("📡 MCP mode: local — using local DB (stdio)"); + // Fall through to local mode + } + } + + // ── Local stdio mode (original behavior) ────────────────────────── + use rmcp::{transport::stdio, ServiceExt}; + + tracing::info!("🚀 Starting codesearch MCP server"); + + // Use database discovery to find the best database + let db_info = find_best_database(path.as_deref())?; + + let (project_path, db_path) = if let Some(info) = db_info { + (info.project_path, info.db_path) + } else { + // No database found + if !may_create_missing_index( + startup.create_index, + startup.readonly, + startup.require_ready, + ) { + return Err(anyhow::anyhow!( + "No database found in current directory, parent directories, or globally tracked repositories. \ + Run 'codesearch index' first, or start writable local MCP with \ + --create-index=true and without --readonly/--require-ready." + )); + } + + // Create minimal database structure to allow server to start immediately + let effective_path = path.as_ref().cloned().unwrap_or(std::env::current_dir()?); + + // Use git root detection to place database in the correct location + let db_root = + crate::index::find_git_root(&effective_path)?.unwrap_or_else(|| effective_path.clone()); + let db_path = db_root.join(".codesearch.db"); + + tracing::info!( + "📁 Creating minimal database structure at {}", + db_path.display() + ); + + // Create directory + std::fs::create_dir_all(&db_path)?; + + // Get model info + let model_type = ModelType::default(); + let model_short_name = model_type.short_name().to_string(); + let dimensions = model_type.dimensions(); + + // Create minimal metadata.json (atomic read-modify-write, matching format + // used by build_index). Routes through the single-source-of-truth stamp so + // model_name matches the other index paths (previously wrote the Debug + // variant name here, e.g. "AllMiniLML6V2Q", instead of the model name). + crate::vectordb::merge_metadata_atomic(&db_path, |obj| { + model_type.write_metadata_fields(obj); + obj.insert( + "indexed_at".to_string(), + serde_json::Value::String(chrono::Utc::now().to_rfc3339()), + ); + })?; + + // Create minimal file_meta.json (matching FileMetaStore format) + let file_meta = crate::cache::FileMetaStore::new(model_short_name.clone(), dimensions); + file_meta.save(&db_path)?; + + // Create FTS directory + let fts_path = db_path.join("fts"); + std::fs::create_dir_all(&fts_path)?; + + // Create LMDB file by opening VectorStore (creates minimal structure) + let _store = crate::vectordb::VectorStore::new(&db_path, dimensions)?; + + tracing::info!("✅ Minimal database created successfully"); + tracing::info!("🔄 Background indexing will begin shortly via incremental refresh"); + + (effective_path, db_path) + }; + + // Initialize file logger now that db_path is known (works for both existing and auto-created DB) + // NOTE: For MCP, tracing is NOT initialized in main.rs — this is the only init call + if let Err(e) = crate::logger::init_logger(&db_path, log_level, quiet) { + tracing::warn!("Failed to initialize file logger: {}", e); + } + + tracing::info!("📂 Project: {}", project_path.display()); + tracing::info!("💾 Database: {}", db_path.display()); + + // Read model metadata to get dimensions (fallback to 384 if missing/corrupt) + let metadata_path = db_path.join("metadata.json"); + let dimensions = if metadata_path.exists() { + match std::fs::read_to_string(&metadata_path) + .ok() + .and_then(|c| serde_json::from_str::(&c).ok()) + .and_then(|j| j.get("dimensions").and_then(|v| v.as_u64())) + { + Some(d) => d as usize, + None => { + tracing::warn!( + "⚠️ Could not parse dimensions from metadata.json, using default {}", + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS + ); + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS + } + } + } else { + tracing::warn!( + "⚠️ metadata.json not found, using default dimensions {}", + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS + ); + crate::constants::DEFAULT_EMBEDDING_DIMENSIONS + }; + + // `--readonly` serves a caller-owned index and never takes the writer lock. + // Otherwise try write mode first and fall back to readonly if it is held, + // so multiple terminal windows can use the same database. + tracing::info!("📦 Creating shared stores..."); + let (shared_stores, is_readonly) = if startup.readonly { + (SharedStores::new_readonly(&db_path, dimensions)?, true) + } else { + SharedStores::new_or_readonly(&db_path, dimensions)? + }; + let shared_stores = Arc::new(shared_stores); + + if startup.readonly { + tracing::info!("📖 Readonly mode requested: the caller owns this index"); + } else if is_readonly { + tracing::warn!("🔒 Running in READONLY mode (another instance has write access)"); + tracing::warn!(" ↳ Searches work normally, but index won't auto-update"); + tracing::warn!(" ↳ Close the other instance to enable write mode"); + } + + // Reject an incomplete index before the transport exists, so the caller sees + // a startup failure instead of a served "building" status it cannot fix. + if startup.require_ready { + require_prebuilt_index_ready(&shared_stores, &db_path).await?; + tracing::info!("✅ Vector and full-text indexes are ready"); + } + + // Create MCP service with shared stores (ready immediately) + let service = CodesearchService::new_with_stores( + Some(project_path.clone()), + Some(shared_stores.clone()), + )?; + + tracing::info!("🧠 Model: {}", service.model_type.name()); + + // START MCP SERVER NOW - fixes timeout! + tracing::info!( + "🚀 Starting MCP server{}...", + if is_readonly { " (readonly)" } else { "" } + ); + let server = service.serve(stdio()).await?; + + tracing::info!("MCP server ready. Waiting for requests..."); + + // Only run background tasks if we have write access + if !is_readonly { + // Create IndexManager with shared stores (skip initial refresh - do in background) + tracing::info!("🔍 Initializing index manager..."); + let index_manager = + IndexManager::new_without_refresh(&project_path, shared_stores.clone()).await?; + + // Background: refresh FIRST, then file watcher (sequential, not concurrent) + // Both write to SharedStores, so they must not run concurrently + let project_path_clone = project_path.clone(); + let db_path_clone = db_path.clone(); + let shared_stores_clone = shared_stores.clone(); + let index_manager_arc = Arc::new(index_manager); + let bg_cancel_token = cancel_token.clone(); + tokio::spawn(async move { + // Step 0: Pre-start FSW to collect file change events during refresh + // This ensures changes made while the refresh is running are not missed + if let Err(e) = index_manager_arc.start_watching().await { + tracing::warn!("⚠️ Could not pre-start file watcher: {}", e); + } + + // Step 1: Run initial refresh (writes to stores) + tracing::info!("🔄 Starting background incremental refresh..."); + match IndexManager::perform_incremental_refresh_with_stores( + &project_path_clone, + &db_path_clone, + &shared_stores_clone, + &bg_cancel_token, + ) + .await + { + Ok(_) => { + tracing::info!("✅ Background incremental refresh completed"); + + // Check if shutdown was requested during refresh + if bg_cancel_token.is_cancelled() { + tracing::info!("🛑 Shutdown requested, skipping file watcher startup"); + return; + } + + // Step 2: AFTER refresh completes, start file watcher (also writes to stores) + tracing::info!("👀 Starting file watcher..."); + if let Err(e) = index_manager_arc + .start_file_watcher(bg_cancel_token, None, None) + .await + { + tracing::error!("❌ Failed to start file watcher: {}", e); + } else { + tracing::info!( + "✅ File watcher active - index will auto-update on file changes" + ); + } + } + Err(e) => { + tracing::error!("❌ Background incremental refresh failed: {}", e); + } + } + }); + + // Start periodic log cleanup task + let db_path_for_cleanup = db_path.clone(); + let cleanup_cancel_token = cancel_token.clone(); + tokio::spawn(async move { + use crate::logger::{cleanup_old_logs, LogRotationConfig}; + + // Run initial cleanup on startup + let rotation_config = LogRotationConfig::from_env(); + tracing::info!("🧹 Running initial log cleanup..."); + if let Err(e) = cleanup_old_logs(&db_path_for_cleanup, &rotation_config) { + tracing::warn!("Initial log cleanup failed: {}", e); + } + + // Start periodic cleanup task (every 24 hours by default) + crate::logger::start_cleanup_task( + db_path_for_cleanup.clone(), + rotation_config, + cleanup_cancel_token, + ); + }); + } else { + tracing::info!("📖 Readonly mode: skipping background refresh and file watcher"); + } + + // Wait for shutdown: either MCP transport closes or cancellation token fires + tokio::select! { + result = server.waiting() => { + tracing::info!("MCP server transport closed"); + result?; + } + _ = cancel_token.cancelled() => { + tracing::info!("🛑 Shutdown signal received, stopping MCP server..."); + } + } + + tracing::info!("✅ MCP server shut down cleanly"); + Ok(()) +} diff --git a/src/mcp/search.rs b/src/mcp/search.rs new file mode 100644 index 00000000..a00499a9 --- /dev/null +++ b/src/mcp/search.rs @@ -0,0 +1,1165 @@ +//! Consolidated `search` tool (semantic/lexical dispatch) + the semantic +//! search machinery it drives. Extracted from `mod.rs` (todo #105) — the +//! `#[tool]` method registers through the per-module router merged in +//! `mod.rs`'s `merged_tool_router`. + +use super::*; +use rmcp::{ + handler::server::wrapper::Parameters, + model::{CallToolResult, ContentBlock}, + tool, tool_router, ErrorData as McpError, +}; + +#[tool_router(router = search_router, vis = "pub(crate)")] +impl CodesearchService { + // Consolidated tools (the primary 5-tool surface) + // ───────────────────────────────────────────────────────────────── + + /// Unified search tool — dispatches to semantic or literal search based on `mode`. + #[tool( + description = "Unified code search. Set `mode` to choose the backend:\n\n- `semantic` (default): vector embeddings + BM25 FTS + exact-identifier boosting, fused with RRF. Best for conceptual queries, identifier lookups, and mixed natural-language + symbol queries.\n- `literal`: pure FTS, no embeddings. Fast and works without an embedding model. Sub-mode selection:\n * Queries with operators, brackets, or punctuation (`foo = null`, `Vec`, `return x;`, `a::b`) -> set `regex=true` and write the query as a regex. BM25 tokenizes on punctuation otherwise, producing noisy results.\n * Multi-word exact phrases -> set `phrase=true`.\n * Plain identifier lookups (`CodesearchService`) -> leave both false.\n\nFor semantic mode, optionally set `semantic_mode`: \"auto\" (default) | \"semantic\" | \"lexical\" | \"hybrid\".\nReturns metadata only by default (`compact=true`). Use `get_chunk` to read full code. Prefer `search(mode=\"literal\", regex=true)` over external grep/ripgrep for code patterns.\n\nIMPORTANT (multi-repo): always specify either `project` (single repo) or `group` (cross-repo). Omitting both in multi-repo mode returns a `scope_required` error with the list of available projects and groups. If the user has not indicated which repository to search, ask them to choose." + )] + pub(crate) async fn search( + &self, + Parameters(request): Parameters, + ) -> Result { + tracing::info!( + "📥 search(query={:?}, mode={:?}, project={:?}, group={:?})", + request.query, + request.mode, + request.project, + request.group, + ); + + // Federation: when the query targets a group that resolves to one or more + // remote peers, merge local + remote results (RRF-interleave) instead of + // searching local repos only. Only `group` federates; `project` stays + // local because project aliases are instance-local. + if let Some(group) = request.group.as_deref() { + let cfg = self.federation_config(); + if Self::group_has_remotes(&cfg, group) { + let remote_projects = cfg.group_remote_projects(group); + return self.federated_search(&request, &cfg, remote_projects).await; + } + } + + // Project-level federation (mounted remote project): a `project` of the + // form "/" transparently routes to that single peer's own + // `` project — a 1-to-1 passthrough, as if the index were local. + // Local repos ALWAYS win a name clash: only route remotely when the name + // does not resolve to a local project. + if let Some(proj) = request.project.as_deref() { + let cfg = self.federation_config(); + if cfg.resolve(proj).is_none() { + if let Some(crate::db_discovery::repos::Target::RemoteProject { + peer_name, + peer, + remote_alias, + }) = cfg.resolve_remote_project(proj) + { + return self + .federated_project_search(&request, peer_name, peer, remote_alias) + .await; + } + } + } + + let mode = request.mode.as_deref().unwrap_or("semantic").to_lowercase(); + match mode.as_str() { + "semantic" => { + // Delegate to the existing semantic_search implementation + let semantic_req = SemanticSearchRequest { + query: request.query, + limit: request.limit, + compact: request.compact, + filter_path: request.filter_path, + mode: request.semantic_mode, + project: request.project, + group: request.group, + }; + self.semantic_search(Parameters(semantic_req)).await + } + "literal" => { + // Delegate to the existing literal_search implementation + let literal_req = LiteralSearchRequest { + query: request.query, + regex: request.regex, + phrase: request.phrase, + limit: request.limit, + file_glob: request.file_glob, + language: request.language, + format: request.format, + project: request.project, + group: request.group, + }; + self.literal_search(Parameters(literal_req)).await + } + _ => Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Unknown search mode '{}'. Use `semantic` or `literal`.", + mode + ))])), + } + } + + // Internal implementations (called by consolidated tools above) + // ───────────────────────────────────────────────────────────────── + + /// Internal: semantic/hybrid search implementation used by `search(mode="semantic")`. + pub(crate) async fn semantic_search( + &self, + Parameters(request): Parameters, + ) -> Result { + // Resolve project/group routing (multi-store for group fan-out) + let ctx = match self + .resolve_routing(&request.project, &request.group, false, "search") + .await + { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + let limit = request.limit.unwrap_or(10); + let compact = request.compact.unwrap_or(true); + let mode = request.mode.as_deref().unwrap_or("auto"); + let identifiers = detect_identifiers(&request.query); + let has_identifiers = !identifiers.is_empty(); + + tracing::debug!( + "MCP semantic_search: query='{}', limit={}, compact={}, mode='{}', multi={}", + request.query, + limit, + compact, + mode, + ctx.is_multi + ); + + // Ensure database exists (skip if serve-mode with routed stores) + if ctx.needs_local_db { + if let Err(e) = self.ensure_database_exists() { + return Ok(CallToolResult::success(vec![ContentBlock::text(e)])); + } + } + + // === Multi-store group fan-out === + if ctx.is_multi { + return self + .semantic_search_multi( + &request, + &identifiers, + limit, + compact, + ctx.stores_vec.unwrap(), + ctx.store_aliases.as_ref().unwrap(), + &ctx.alias_roots, + ) + .await; + } + + // === Mode: "lexical" — FTS only, no embedding === + if mode == "lexical" { + tracing::debug!("MCP: mode=lexical — skipping embedding service"); + return self + .semantic_search_lexical( + &request, + &identifiers, + limit, + compact, + ctx.stores, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + ) + .await; + } + + // === Modes: "semantic", "hybrid", "auto" — require embedding === + // The query MUST be embedded with the model the target index was built + // with. In serve mode that is the routed repo's recorded model, not a + // hub-wide default: a 384-dim query against a 768-dim EmbeddingGemma + // index failed with "expected 768, got 384". A repo that records no + // model is queried with the built-in default, and the caller is warned. + let model_resolution = self.resolve_query_model(ctx.project_alias.as_deref()); + let query_embedding = { + let model = model_resolution.model; + let service = match self.embedding_service_for(model) { + Ok(s) => s, + Err(e) => { + tracing::error!("MCP: Failed to get embedding service: {:?}", e); + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error initializing embedding service: {e:#}" + ))])); + } + }; + + let mut service = service.lock().unwrap(); + tracing::debug!( + "MCP: Embedding query with model '{}'...", + model.short_name() + ); + match service.embed_query(&request.query) { + Ok(e) => e, + Err(e) => { + tracing::error!("MCP: Failed to embed query: {:?}", e); + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error embedding query: {e:#}" + ))])); + } + } + }; + + // Failures on this single-store path. The group fan-out has carried a + // warnings channel since the read-only incident; without the same thing + // here, `project=` — the form an agent uses most — still reports + // a broken store as an ordinary empty result. + let mut single_warnings: Vec = Vec::new(); + // Surface the assumed-model warning even when the store read succeeds: + // mismatched vector spaces do not error, they just rank wrongly. + if let Some(warning) = model_resolution.assumed_warning { + single_warnings.push(warning); + } + + // Search vector store + let vector_results = match self + .with_vector_store_read_for( + |store| { + store + .search(&query_embedding, limit * 5) + .context("Error searching vector store") + }, + ctx.stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + tracing::error!("MCP: Search failed: {:?}", e); + // Only "semantic" has no second backend to fall back on. In + // hybrid/auto the FTS half can still answer, so hard-failing + // here would throw away good results — the same mistake this + // branch already fixed once in the group fan-out. + // + // `{:#}` renders the whole anyhow chain. With plain `{}` the + // caller only ever saw the outermost `.context(...)` wrapper + // ("Error reading from project-routed vector store"), which + // hides the actual fault and makes remote diagnosis guesswork. + if mode == "semantic" { + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching vector store: {:#}", + e + ))])); + } + single_warnings.push(format!("vector search failed: {e:#}")); + Vec::new() + } + }; + + tracing::debug!("MCP: Found {} vector results", vector_results.len()); + + // === Mode: "semantic" — vector only, skip FTS fusion === + if mode == "semantic" { + tracing::debug!("MCP: mode=semantic — using vector results only"); + let fused = vector_only(&vector_results); + + let chunk_to_result: std::collections::HashMap = + vector_results.iter().map(|r| (r.id, r)).collect(); + + let mut results: Vec = Vec::new(); + for f in fused.into_iter().take(limit) { + if let Some(result) = chunk_to_result.get(&f.chunk_id) { + let mut r = (*result).clone(); + r.score = f.rrf_score; + results.push(r); + } + } + return self.build_semantic_response( + results, + &request, + compact, + has_identifiers, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + &single_warnings, + ); + } + + // === Modes: "hybrid" | "auto" — full hybrid search === + let structural_intent = detect_structural_intent(&request.query); + let (vector_k, fts_k) = adapt_rrf_k(&request.query); + + tracing::debug!( + "MCP: Query analysis - identifiers: {:?}, structural_intent: {:?}, rrf_k: ({}, {})", + identifiers, + structural_intent, + vector_k, + fts_k + ); + + // Perform FTS search and fusion + let mut results = match self + .with_fts_store_read_for( + |fts_store| { + let fts_results = fts_store + .search(&request.query, limit * 5, structural_intent) + .context("Error searching FTS store")?; + + let fused = if identifiers.is_empty() { + rrf_fusion(&vector_results, &fts_results, vector_k as f32) + } else { + let mut all_exact: Vec = Vec::new(); + for ident in &identifiers { + if let Ok(exact) = + fts_store.search_exact(ident, limit * 3, structural_intent) + { + for r in exact { + if !all_exact.iter().any(|e| e.chunk_id == r.chunk_id) { + all_exact.push(r); + } + } + } + } + + tracing::debug!( + "MCP: FTS found {} results, exact found {} results", + fts_results.len(), + all_exact.len() + ); + + rrf_fusion_with_exact( + &vector_results, + &fts_results, + &all_exact, + vector_k as f32, + fts_k as f32, + EXACT_MATCH_RRF_K, + ) + }; + + Ok(fused) + }, + ctx.stores.clone(), + ) + .await + { + Ok(fused) => { + // Map FusedResult back to SearchResult + let chunk_to_result: std::collections::HashMap< + u32, + &crate::vectordb::SearchResult, + > = vector_results.iter().map(|r| (r.id, r)).collect(); + + let mut mapped: Vec = Vec::new(); + for f in fused.into_iter().take(limit) { + if let Some(result) = chunk_to_result.get(&f.chunk_id) { + let mut r = (*result).clone(); + r.score = f.rrf_score; + mapped.push(r); + } + } + mapped + } + Err(e) => { + tracing::warn!("MCP: FTS store unavailable, using vector-only: {:?}", e); + // Degrading to vector-only is correct, but it must be VISIBLE: + // a caller that gets half a hybrid search with no signal cannot + // tell it from a complete one. + single_warnings.push(format!("lexical (FTS) search failed: {e:#}")); + vector_results.into_iter().take(limit).collect() + } + }; + + // Apply language boost + if let Some((_, _, Some(primary_lang))) = crate::search::read_metadata(&self.db_path) { + for result in &mut results { + let file_lang = format!( + "{:?}", + Language::from_path(std::path::Path::new(&result.path)) + ); + if file_lang.to_lowercase() == primary_lang.to_lowercase() { + result.score *= 1.2; + } + } + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } + + // Apply kind boost + if let Some(target_kind) = structural_intent { + boost_kind(&mut results, target_kind); + } + + // Auto-fallback: if hybrid search returned very few results for a code-like query, + // run literal FTS and merge missing chunks. + if results.len() < 3 && has_identifiers { + tracing::debug!( + "Auto-fallback: semantic returned {} results, trying literal", + results.len() + ); + + let literal_results = self + .with_fts_store_read_for( + |fts_store| fts_store.search(&request.query, limit, None), + ctx.stores.clone(), + ) + .await + .unwrap_or_default(); + + let mut existing_ids: std::collections::HashSet = + results.iter().map(|r| r.id).collect(); + + for fts in literal_results { + if results.len() >= limit { + break; + } + if existing_ids.contains(&fts.chunk_id) { + continue; + } + + let maybe_resolved = match self + .with_vector_store_read_for( + |store| { + // `Ok(None)` means "this store does not hold that + // chunk" — a normal miss to skip. `Err` means the + // store is broken and must propagate: flattening + // the two silently dropped every remaining literal + // hit whenever the vector store was down, turning a + // dead store into an ordinary-looking short result. + let chunk = match store.get_chunk(fts.chunk_id)? { + Some(c) => c, + None => return Ok(None), + }; + Ok(Some(crate::vectordb::SearchResult { + id: fts.chunk_id, + content: chunk.content, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + docstring: chunk.docstring, + context: chunk.context, + hash: chunk.hash, + distance: 0.0, + score: fts.score, + context_prev: chunk.context_prev, + context_next: chunk.context_next, + })) + }, + ctx.stores.clone(), + ) + .await + { + Ok(resolved) => resolved, + Err(e) => { + // The old `.ok()` here folded a dead store into "no + // more literal hits" with zero signal to the caller — + // the exact false negative `single_warnings` exists + // for. Note it and stop: every further lookup against + // this store would fail the same way. + single_warnings.push(format!("literal-hit chunk lookup failed: {e:#}")); + break; + } + }; + + if let Some(resolved) = maybe_resolved { + existing_ids.insert(resolved.id); + results.push(resolved); + } + } + } + + tracing::debug!("MCP: Final {} results after hybrid search", results.len()); + self.build_semantic_response( + results, + &request, + compact, + has_identifiers, + ctx.project_alias.as_deref(), + &ctx.alias_roots, + &single_warnings, + ) + } + + // === Helper methods (not exposed as tools) === + + /// Multi-store semantic search: fan out across all stores, merge raw vector/FTS + /// results, then apply RRF fusion. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn semantic_search_multi( + &self, + request: &SemanticSearchRequest, + identifiers: &[String], + limit: usize, + compact: bool, + stores: Vec>, + aliases: &[String], + alias_roots: &std::collections::HashMap, + ) -> Result { + let mode = request.mode.as_deref().unwrap_or("auto"); + let structural_intent = detect_structural_intent(&request.query); + + // === Lexical mode: FTS only across all stores === + if mode == "lexical" { + // Lexical has no second backend, so a failed store here is invisible + // unless it is reported: the query simply looks like it found nothing. + let mut lexical_warnings: Vec = Vec::new(); + + let outcome = self + .with_fts_store_read_multi( + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), + stores.clone(), + aliases, + ) + .await + .unwrap_or_default(); + if !outcome.failures.is_empty() { + tracing::error!( + "MCP: lexical fan-out degraded — {} of {} repo(s) failed: {:?}", + outcome.failures.len(), + stores.len(), + outcome.failures + ); + lexical_warnings.extend(outcome.warnings("literal search")); + } + let fts_results = outcome.results; + + // Also do exact search if identifiers detected + let mut all_fts = fts_results; + for ident in identifiers { + let exact_outcome = self + .with_fts_store_read_multi( + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), + stores.clone(), + aliases, + ) + .await + .unwrap_or_default(); + lexical_warnings.extend(exact_outcome.warnings("exact-identifier search")); + merge_exact_into_fts(&mut all_fts, exact_outcome.results); + } + + all_fts.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let results = self + .resolve_fts_to_search_results_multi( + &all_fts, + limit, + &stores, + aliases, + &mut lexical_warnings, + ) + .await; + + if let Some(target_kind) = structural_intent { + // We need mutable results but we have them as vectordb::SearchResult + let mut mutable_results = results; + boost_kind(&mut mutable_results, target_kind); + return self.build_semantic_response( + mutable_results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + &lexical_warnings, + ); + } + + return self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + &lexical_warnings, + ); + } + + // === Modes requiring embedding: "semantic", "hybrid", "auto" === + // + // Each repo may have been indexed with a different model, so the query + // is embedded once per distinct model and every store is searched with + // the embedding of ITS model. Embedding all stores with one hub-wide + // default is what produced "Query embedding dimension mismatch: + // expected 768, got 384" on a mixed hub. + let mut embeddings_by_alias: std::collections::HashMap> = + std::collections::HashMap::with_capacity(aliases.len()); + // Assumed-model warnings, one per repo that records no model. Collected + // here and folded into `search_warnings` below so an agent sees the + // assumption alongside the results it applies to. + let mut model_warnings: Vec = Vec::new(); + { + let mut by_model: std::collections::HashMap> = + std::collections::HashMap::new(); + for alias in aliases { + let model_resolution = self.resolve_query_model(Some(alias)); + let model = model_resolution.model; + if let Some(warning) = model_resolution.assumed_warning { + model_warnings.push(warning); + } + let embedding = match by_model.get(&model) { + Some(cached) => cached.clone(), + None => { + let service = match self.embedding_service_for(model) { + Ok(s) => s, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text( + format!( + "Error initializing embedding service for '{alias}': {e:#}" + ), + )])); + } + }; + let mut service = service.lock().unwrap(); + let embedding = match service.embed_query(&request.query) { + Ok(e) => e, + Err(e) => { + return Ok(CallToolResult::success(vec![ContentBlock::text( + format!("Error embedding query: {e:#}"), + )])); + } + }; + by_model.insert(model, embedding.clone()); + embedding + } + }; + embeddings_by_alias.insert(alias.clone(), embedding); + } + } + + // Search vector stores across all repos, each with its own model's + // query embedding. + let outcome = self + .with_vector_store_read_multi( + |alias, store| { + let embedding = embeddings_by_alias.get(alias).ok_or_else(|| { + anyhow::anyhow!( + "internal error: no query embedding resolved for repo '{alias}'" + ) + })?; + store + .search(embedding, limit * 5) + .context("Error searching vector store") + }, + stores.clone(), + aliases, + ) + .await; + + // Warnings raised by the fan-out, carried into the response so the + // calling agent can tell "not in the corpus" from "that repo is down". + // Seeded with any assumed-model warnings gathered while embedding. + let mut search_warnings: Vec = model_warnings; + + let vector_results = + match outcome { + Ok(o) => { + if !o.failures.is_empty() { + tracing::error!( + "MCP: vector fan-out degraded — {} of {} repo(s) failed: {:?}", + o.failures.len(), + stores.len(), + o.failures + ); + // Only "semantic" has no second backend to fall back on. In + // hybrid/auto/lexical the FTS half can still answer, so + // hard-failing here would throw away good results — the same + // reason one broken repo does not abort the whole fan-out. + if mode == "semantic" && o.results.is_empty() { + let detail = o + .failures + .iter() + .map(|(alias, err)| format!(" - {alias}: {err}")) + .collect::>() + .join("\n"); + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching vector store: {} of {} repo(s) in scope failed \ + and none returned results:\n{}", + o.failures.len(), + stores.len(), + detail + ))])); + } + search_warnings.extend(o.failures.iter().map(|(alias, err)| { + format!("repo '{alias}' vector search failed: {err}") + })); + } + o.results + } + Err(e) => { + tracing::error!("MCP: vector fan-out failed: {:?}", e); + return Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Error searching vector store: {e:#}" + ))])); + } + }; + + // === Mode: "semantic" — vector only === + if mode == "semantic" { + let fused = vector_only(&vector_results); + let chunk_to_result: std::collections::HashMap = + vector_results.iter().map(|r| (r.id, r)).collect(); + + let mut results: Vec = Vec::new(); + for f in fused.into_iter().take(limit) { + if let Some(result) = chunk_to_result.get(&f.chunk_id) { + let mut r = (*result).clone(); + r.score = f.rrf_score; + results.push(r); + } + } + return self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + &search_warnings, + ); + } + + // === Modes: "hybrid" | "auto" — full hybrid search === + let (vector_k, fts_k) = adapt_rrf_k(&request.query); + + // FTS search across all stores. Its failures matter as much as the + // vector half's: during the cloud read-only incident literal search + // also returned 0 results for every affected vendor, and looked clean. + let fts_outcome = self + .with_fts_store_read_multi( + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), + stores.clone(), + aliases, + ) + .await + .unwrap_or_default(); + if !fts_outcome.failures.is_empty() { + tracing::error!( + "MCP: FTS fan-out degraded — {} of {} repo(s) failed: {:?}", + fts_outcome.failures.len(), + stores.len(), + fts_outcome.failures + ); + search_warnings.extend(fts_outcome.warnings("literal search")); + } + let fts_results = fts_outcome.results; + + // Exact identifier search across all stores + let all_exact = if !identifiers.is_empty() { + let mut exact_results: Vec = Vec::new(); + for ident in identifiers { + let exact_outcome = self + .with_fts_store_read_multi( + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), + stores.clone(), + aliases, + ) + .await + .unwrap_or_default(); + search_warnings.extend(exact_outcome.warnings("exact-identifier search")); + for r in exact_outcome.results { + if !exact_results.iter().any(|e| e.chunk_id == r.chunk_id) { + exact_results.push(r); + } + } + } + exact_results + } else { + Vec::new() + }; + + // RRF fusion + let fused = if identifiers.is_empty() { + rrf_fusion(&vector_results, &fts_results, vector_k as f32) + } else { + rrf_fusion_with_exact( + &vector_results, + &fts_results, + &all_exact, + vector_k as f32, + fts_k as f32, + EXACT_MATCH_RRF_K, + ) + }; + + // Map FusedResult back to SearchResult via chunk lookup across all stores + let chunk_to_result: std::collections::HashMap = + vector_results.iter().map(|r| (r.id, r)).collect(); + + let mut mapped: Vec = Vec::new(); + for f in fused.into_iter().take(limit) { + if let Some(result) = chunk_to_result.get(&f.chunk_id) { + let mut r = (*result).clone(); + r.score = f.rrf_score; + mapped.push(r); + } else { + // Chunk from FTS but not in vector results — resolve from stores + if let Some(resolved) = self + .resolve_chunk_from_stores( + f.chunk_id, + f.rrf_score, + &stores, + aliases, + &mut search_warnings, + ) + .await + { + mapped.push(resolved); + } + } + } + + // Apply kind boost + if let Some(target_kind) = structural_intent { + boost_kind(&mut mapped, target_kind); + } + + self.build_semantic_response( + mapped, + request, + compact, + !identifiers.is_empty(), + None, + alias_roots, + &search_warnings, + ) + } + + /// Resolve a single chunk from multiple stores (used for FTS-only hits in multi-store fusion). + async fn resolve_chunk_from_stores( + &self, + chunk_id: u32, + score: f32, + stores: &[Arc], + aliases: &[String], + warnings: &mut Vec, + ) -> Option { + for (idx, store_arc) in stores.iter().enumerate() { + let store = store_arc.vector_store.read().await; + let looked_up = store.get_chunk(chunk_id); + if let Err(ref e) = looked_up { + note_store_failure(warnings, aliases, idx, "chunk lookup", e); + } + if let Ok(Some(chunk)) = looked_up { + return Some(crate::vectordb::SearchResult { + id: chunk_id, + content: chunk.content, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + docstring: chunk.docstring, + context: chunk.context, + hash: chunk.hash, + distance: 0.0, + score, + context_prev: chunk.context_prev, + context_next: chunk.context_next, + }); + } + } + None + } + + /// Resolve FTS results to SearchResult using multiple stores. + async fn resolve_fts_to_search_results_multi( + &self, + fts_results: &[crate::fts::FtsResult], + limit: usize, + stores: &[Arc], + aliases: &[String], + warnings: &mut Vec, + ) -> Vec { + let mut results = Vec::new(); + for fts in fts_results.iter().take(limit) { + for (idx, store_arc) in stores.iter().enumerate() { + let store = store_arc.vector_store.read().await; + let looked_up = store.get_chunk(fts.chunk_id); + if let Err(ref e) = looked_up { + // `Ok(None)` means "this store does not hold that chunk" and + // is normal during fan-out; `Err` means the store is broken. + // Collapsing the two is how a dead vector store renders as + // an empty literal search — the exact shape of the step-8 + // incident, which tantivy-side checks cannot detect. + note_store_failure(warnings, aliases, idx, "chunk lookup", e); + } + if let Ok(Some(chunk)) = looked_up { + results.push(crate::vectordb::SearchResult { + id: fts.chunk_id, + content: chunk.content, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + docstring: chunk.docstring, + context: chunk.context, + hash: chunk.hash, + distance: 0.0, + score: fts.score, + context_prev: chunk.context_prev, + context_next: chunk.context_next, + }); + break; // Found in this store, skip remaining stores + } + } + } + results + } + + /// Lexical-only search: FTS without embedding service. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn semantic_search_lexical( + &self, + request: &SemanticSearchRequest, + identifiers: &[String], + limit: usize, + compact: bool, + stores: Option>, + project_alias: Option<&str>, + alias_roots: &std::collections::HashMap, + ) -> Result { + let structural_intent = detect_structural_intent(&request.query); + + // `project=`-scoped queries route here, not through the fan-out + // (`is_multi` requires >1 store), so this path needs the same failure + // reporting — it is at least as common as a group query. + let mut lexical_warnings: Vec = Vec::new(); + + let mut fts_results = match self + .with_fts_store_read_for( + |fts_store| fts_store.search(&request.query, limit * 5, structural_intent), + stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + let msg = format!("literal search failed: {e:#}"); + tracing::error!("MCP: {}", msg); + lexical_warnings.push(msg); + Vec::new() + } + }; + + // Also do exact search if identifiers detected + for ident in identifiers { + let exact = match self + .with_fts_store_read_for( + |fts_store| fts_store.search_exact(ident, limit * 3, structural_intent), + stores.clone(), + ) + .await + { + Ok(r) => r, + Err(e) => { + let msg = format!("exact-identifier search for '{ident}' failed: {e:#}"); + tracing::error!("MCP: {}", msg); + lexical_warnings.push(msg); + continue; + } + }; + merge_exact_into_fts(&mut fts_results, exact); + } + + fts_results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + // Resolve FTS results to chunk metadata + let mut results = self + .resolve_fts_to_search_results(&fts_results, limit, stores, &mut lexical_warnings) + .await; + + // Apply kind boost + if let Some(target_kind) = structural_intent { + boost_kind(&mut results, target_kind); + } + + self.build_semantic_response( + results, + request, + compact, + !identifiers.is_empty(), + project_alias, + alias_roots, + &lexical_warnings, + ) + } + + /// Build the final SemanticSearchResponse with low-confidence signaling. + // Eight parameters, one over clippy's threshold. Bundling them into a + // `ResponseContext` struct is the right end state and is recorded as a + // follow-up; doing it in an incident fix would touch all seven call sites + // for no behavioural gain. The alternative — dropping `warnings` — is not + // acceptable: without it a failed repo is silently reported as "no match". + #[allow(clippy::too_many_arguments)] + fn build_semantic_response( + &self, + results: Vec, + request: &SemanticSearchRequest, + compact: bool, + has_identifiers: bool, + project_alias: Option<&str>, + alias_roots: &std::collections::HashMap, + // Repos that failed during a fan-out. MUST reach the caller: the + // consumer of this tool is a remote agent that never sees the server + // log, so a silently omitted repo reads as "no match there" — a false + // negative. The federated path already does this (`warnings` on the + // remote-project fan-out); the local path never could. + warnings: &[String], + ) -> Result { + let warnings = if warnings.is_empty() { + None + } else { + Some(warnings.to_vec()) + }; + if results.is_empty() { + let response = SemanticSearchResponse { + results: vec![], + low_confidence: Some(true), + suggested_tool: retry_hint(Some("literal_search".to_string()), &warnings), + warnings, + }; + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + + // Pre-compute normalized project root for stripping absolute paths + let project_root_normalized = { + let root = crate::cache::normalize_path_str(self.project_path.to_str().unwrap_or("")); + root.trim_end_matches('/').to_string() + }; + + let mut items: Vec = results + .into_iter() + .filter(|r| { + if let Some(ref fp) = request.filter_path { + let normalized_filter = crate::cache::normalize_filter_path(fp); + if normalized_filter.is_empty() { + return true; + } + // Relativise against the ROUTED project's root, not the + // service's own project_path — otherwise a serve-routed + // absolute path never strips and every hit is dropped. + let filter_root = pick_filter_root( + &r.path, + project_alias, + alias_roots, + &project_root_normalized, + ); + crate::cache::path_matches_filter(&r.path, &normalized_filter, &filter_root) + } else { + true + } + }) + .map(|r| SearchResultItem { + chunk_id: Some(r.id), + path: r.path, + start_line: r.start_line, + end_line: r.end_line, + kind: r.kind, + score: r.score, + signature: r.signature, + content: if compact { None } else { Some(r.content) }, + context_prev: if compact { None } else { r.context_prev }, + context_next: if compact { None } else { r.context_next }, + source: None, + chunk_ref: None, + }) + .collect(); + + // Prefix paths with alias for multi-repo / single-project identification + for item in &mut items { + if let Some(alias) = project_alias { + if let Some(root) = alias_roots.get(alias) { + item.path = prefix_path_with_alias(&item.path, Some(alias), root); + } else { + item.path = crate::cache::normalize_path_str(&item.path); + } + } else if !alias_roots.is_empty() { + item.path = prefix_path_multi(&item.path, &[], alias_roots); + } + } + + // Check low-confidence: top result's RRF score below threshold + let top_score = items.first().map(|r| r.score); + let (low_confidence, suggested_tool) = compute_low_confidence(top_score, has_identifiers); + let suggested_tool = retry_hint(suggested_tool, &warnings); + + let response = SemanticSearchResponse { + results: items, + low_confidence, + suggested_tool, + warnings, + }; + + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } + + /// Resolve FTS results to SearchResult by looking up chunk metadata. + async fn resolve_fts_to_search_results( + &self, + fts_results: &[crate::fts::FtsResult], + limit: usize, + stores: Option>, + warnings: &mut Vec, + ) -> Vec { + let outcome = self + .with_vector_store_read_for( + |store| { + let mut results = Vec::new(); + for fts in fts_results.iter().take(limit) { + // A failed lookup is not an absent chunk. Propagating the + // error keeps a broken vector store from rendering as an + // ordinary empty literal search. + let chunk = store + .get_chunk(fts.chunk_id) + .context("Error resolving FTS hit to chunk metadata")?; + if let Some(chunk) = chunk { + results.push(crate::vectordb::SearchResult { + id: fts.chunk_id, + content: chunk.content, + path: chunk.path, + start_line: chunk.start_line, + end_line: chunk.end_line, + kind: chunk.kind, + signature: chunk.signature, + docstring: chunk.docstring, + context: chunk.context, + hash: chunk.hash, + distance: 0.0, + score: fts.score, + context_prev: chunk.context_prev, + context_next: chunk.context_next, + }); + } + } + Ok(results) + }, + stores, + ) + .await; + match outcome { + Ok(results) => results, + Err(e) => { + let msg = format!("literal search could not read the index: {e:#}"); + tracing::error!("MCP: {}", msg); + if !warnings.contains(&msg) { + warnings.push(msg); + } + Vec::new() + } + } + } +} diff --git a/src/mcp/status.rs b/src/mcp/status.rs new file mode 100644 index 00000000..969f3d4c --- /dev/null +++ b/src/mcp/status.rs @@ -0,0 +1,457 @@ +//! Consolidated `status` tool (index/projects dispatch) + the status +//! internals. Extracted from `mod.rs` (todo #105) — the `#[tool]` method +//! registers through the per-module router merged in `mod.rs`'s +//! `merged_tool_router`. + +use super::*; +use rmcp::{ + handler::server::wrapper::Parameters, + model::{CallToolResult, ContentBlock}, + tool, tool_router, ErrorData as McpError, +}; + +#[tool_router(router = status_router, vis = "pub(crate)")] +impl CodesearchService { + /// Unified status tool — dispatches based on `kind`. + #[tool( + description = "Unified status/info tool. Set `kind` to choose the action:\n\n- `index` (default): get the status of the local search index (model info, chunk count, readiness)\n- `projects`: list all registered projects/repositories, groups, and their index status" + )] + pub(crate) async fn status( + &self, + Parameters(request): Parameters, + ) -> Result { + let kind = request.kind.as_deref().unwrap_or("index").to_lowercase(); + tracing::info!("📥 status(kind={})", kind); + match kind.as_str() { + "index" => self.index_status_impl(request.project, request.group).await, + "projects" => self.list_projects().await, + _ => Ok(CallToolResult::success(vec![ContentBlock::text(format!( + "Unknown status kind '{}'. Use `index` or `projects`.", + kind + ))])), + } + } + + /// Model label for a grouped index-status response: the common model when + /// every member agrees, `"mixed"` when a hub holds indexes built with + /// different models. + /// + /// The status `model` field used to be the service's own model — the + /// hardcoded default in serve mode — so every repo read as `minilm-l6-q` + /// even when indexed with EmbeddingGemma. See the serve query-model fix. + pub(crate) fn group_model_label(&self, aliases: &[String]) -> String { + let mut common: Option = None; + for alias in aliases { + let model = self.query_model(Some(alias)); + match common { + None => common = Some(model), + Some(prev) if prev != model => return "mixed".to_string(), + _ => {} + } + } + common + .unwrap_or_else(|| self.query_model(None)) + .short_name() + .to_string() + } + + // ───────────────────────────────────────────────────────────────── + /// Internal implementation for index_status with optional project/group routing. + async fn index_status_impl( + &self, + project: Option, + group: Option, + ) -> Result { + // When no project/group specified in serve mode, return lightweight aggregated + // status WITHOUT opening any databases. Only a specific project/group request + // should trigger DB activation. + if project.is_none() && group.is_none() { + if let Some(ref serve_state) = self.serve_state { + let config = serve_state.config_snapshot(); + let repo_count = config.repos.len(); + // Count the virtual "all" group when repos are registered, so the + // summary doesn't read "0 group(s)" while `all` is actually available. + let group_count = config.groups.len() + if config.repos.is_empty() { 0 } else { 1 }; + let statuses = serve_state.repo_statuses_lightweight(); + let open_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Open)) + .count(); + let warm_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Warm)) + .count(); + let closed_count = statuses + .iter() + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Closed)) + .count(); + + let status = if open_count + warm_count > 0 { + "ready".to_string() + } else if repo_count > 0 { + "idle".to_string() + } else { + "no_repos".to_string() + }; + + let status_message = format!( + "{} repo(s) registered, {} group(s). Open: {}, Warm: {}, Closed: {}.", + repo_count, group_count, open_count, warm_count, closed_count + ); + + let response = IndexStatusResponse { + indexed: open_count + warm_count > 0, + status, + status_message, + total_chunks: 0, // Not available without opening DBs + total_files: 0, + model: self.model_type.short_name().to_string(), + dimensions: 0, + max_chunk_id: 0, + db_path: format!("({} repos)", repo_count), + project_path: format!("serve mode — {} repo(s)", repo_count), + error_message: None, + mode: self.mcp_mode(), + }; + + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + } + + // Resolve project/group routing — status is scope-free, allow unscoped fan-out + let ctx = match self.resolve_routing(&project, &group, true, "status").await { + Ok(c) => c, + Err(e) => return Ok(CallToolResult::success(vec![ContentBlock::text(e)])), + }; + + if ctx.needs_local_db { + let indexed = self.db_path.exists(); + + if !indexed { + let response = IndexStatusResponse { + indexed: false, + status: "not_indexed".to_string(), + status_message: "No index found. Run 'codesearch index' or start with --create-index=true to automatically create one.".to_string(), + total_chunks: 0, + total_files: 0, + model: "none".to_string(), + dimensions: 0, + max_chunk_id: 0, + db_path: self.db_path.display().to_string(), + project_path: self.project_path.display().to_string(), + error_message: None, + mode: self.mcp_mode(), + }; + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + } + + if let Some(ref sv) = ctx.stores_vec { + // Multi-store: aggregate stats across all group members + let mut total_chunks = 0usize; + let mut total_files = 0usize; + let mut max_chunk_id = 0u32; + let mut dimensions = 0usize; + let mut all_indexed = true; + // Separate from `all_indexed`: a store that FAILED to report stats + // must not make the summary claim "not built" (the failure already + // rides the `warnings` channel). This tracks only the graph state of + // stores that answered. + let mut all_built = true; + let aliases = ctx.aliases(); + let mut stats_warnings: Vec = Vec::new(); + let mut failed_count = 0usize; + + for (i, store_arc) in sv.iter().enumerate() { + let store = store_arc.vector_store.read().await; + match store.stats() { + Ok(stats) => { + total_chunks += stats.total_chunks; + total_files += stats.total_files; + if stats.max_chunk_id > max_chunk_id { + max_chunk_id = stats.max_chunk_id; + } + if stats.dimensions > 0 { + dimensions = stats.dimensions; + } + if !stats.indexed { + all_indexed = false; + all_built = false; + } + } + // `all_indexed = false` alone renders identically to "still + // warming" — the caller has no way to tell "wait" from "this + // store is down". This is the tool whose job is reporting index + // health, so it must not stay silent on the one signal that + // matters here: bind the error, carry it, never `Err(_)`. + Err(ref e) => { + all_indexed = false; + failed_count += 1; + note_store_failure(&mut stats_warnings, aliases, i, "stats", e); + } + } + } + + let (status, status_message) = + index_status_summary(sv.len(), failed_count, total_chunks, all_built); + + let response = IndexStatusResponse { + indexed: all_indexed, + status, + status_message, + total_chunks, + total_files, + model: self.group_model_label(ctx.aliases()), + dimensions, + max_chunk_id, + db_path: format!("({} repos)", sv.len()), + project_path: format!("group with {} repo(s)", sv.len()), + error_message: None, + mode: self.mcp_mode(), + }; + + return respond_with_object(&response, &stats_warnings); + } + + // Single-store path + let stats = match self + .with_vector_store_read_for( + |store| store.stats().context("Error getting index stats"), + ctx.stores.clone(), + ) + .await + { + Ok(s) => s, + Err(e) => { + let response = IndexStatusResponse { + indexed: false, + status: "error".to_string(), + status_message: format!("{}", e), + total_chunks: 0, + total_files: 0, + model: self + .query_model(ctx.project_alias.as_deref()) + .short_name() + .to_string(), + dimensions: 0, + max_chunk_id: 0, + db_path: self.db_path.display().to_string(), + project_path: self.project_path.display().to_string(), + error_message: Some(format!("{}", e)), + mode: self.mcp_mode(), + }; + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); + } + }; + + // Determine status based on database state. `stats.indexed` (the HNSW + // graph is built) is load-bearing — see `single_index_status`. + let (status, status_message) = single_index_status(stats.total_chunks, stats.indexed); + + let response = IndexStatusResponse { + indexed: stats.indexed, + status, + status_message, + total_chunks: stats.total_chunks, + total_files: stats.total_files, + model: self + .query_model(ctx.project_alias.as_deref()) + .short_name() + .to_string(), + dimensions: stats.dimensions, + max_chunk_id: stats.max_chunk_id, + db_path: self.db_path.display().to_string(), + project_path: self.project_path.display().to_string(), + error_message: None, + mode: self.mcp_mode(), + }; + + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } + + /// List all registered projects and groups. Called by `status(kind="projects")`. + /// Build the `remote_projects` listing (opt-in mounts) for `list_projects`. + fn remote_projects_listing( + config: &crate::db_discovery::repos::ReposConfig, + ) -> Vec { + config + .mounted_remote_projects() + .into_iter() + .filter_map(|(name, target)| match target { + crate::db_discovery::repos::Target::RemoteProject { + peer_name, + peer, + remote_alias, + } => Some(RemoteProjectInfo { + name, + peer: peer_name, + remote_alias, + peer_url: peer.url, + }), + _ => None, + }) + .collect() + } + + async fn list_projects(&self) -> Result { + let current_dir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + + let serve_active = self.serve_state.is_some(); + let serve_url = if serve_active { + Some(serve_url_from_env()) + } else { + None + }; + + // When serve is active, use ServeState as source of truth for lock status + if let Some(ref serve_state) = self.serve_state { + let config = serve_state.config_snapshot(); + let project_groups = config.project_groups(); + let mut repos_info = Vec::new(); + let mut list_warnings: Vec = Vec::new(); + + for (alias, path) in &config.repos { + let db_path = path.join(crate::constants::DB_DIR_NAME); + + let (total_chunks, total_files, model, lock_status, error) = if db_path.exists() { + let (model_name, _dims) = read_model_metadata(&db_path); + + // For repos already opened in DashMap, use the live SharedStores for stats + // WITHOUT opening a new VectorStore connection. + // For unopened repos, just report metadata — do NOT open the DB. + if let Some(stores) = serve_state.get_opened_stores(alias) { + let stats_result = { + let vs = stores.vector_store.read().await; + vs.stats() + }; + // `0 chunks` alone reads exactly like "not indexed yet" — the + // repo may in fact be full and simply failing to answer (the + // read-only-incident shape this branch exists for). Attribute + // the failure to THIS repo rather than a top-level channel: + // list_projects returns one entry per repo, so per-item is the + // shape that actually matches the fan-out. + // `repo_stats_from_result` carries only the part of this + // decision that varies by Ok/Err — see its doc comment. + // `record_stats_or_warn` wraps it so this call site cannot + // silently drop the warning half without also breaking the + // counts it returns — see its own doc comment. + let (total_chunks, total_files, error) = + record_stats_or_warn(stats_result, alias, &mut list_warnings); + ( + total_chunks, + total_files, + model_name, + serve_state + .repo_lock_status(alias) + .unwrap_or("unknown") + .to_string(), + error, + ) + } else { + // Repo NOT opened — read persisted stats from metadata.json + let (md_chunks, md_files) = read_metadata_stats(&db_path); + let lock_status = if crate::index::is_database_locked(&db_path) { + "locked-externally".to_string() + } else { + "available".to_string() + }; + (md_chunks, md_files, model_name, lock_status, None) + } + } else { + (0, 0, "not indexed".to_string(), "unknown".to_string(), None) + }; + + repos_info.push(RepoInfo { + alias: alias.clone(), + project_path: path.display().to_string(), + database_path: db_path.display().to_string(), + total_chunks, + total_files, + model, + lock_status, + groups: project_groups.get(alias).cloned().unwrap_or_default(), + error, + }); + } + + let response = ListProjectsResponse { + repos: repos_info, + groups: config.groups_with_virtual_all(), + remote_projects: Self::remote_projects_listing(&config), + serve_active, + serve_url, + current_directory: current_dir.display().to_string(), + }; + + return respond_with_object(&response, &list_warnings); + } + + // Stdio mode: fall back to disk-based lock detection + let config = load_repos_config().unwrap_or_default(); + let project_groups = config.project_groups(); + let mut repos_info = Vec::new(); + for (alias, path) in &config.repos { + let db_path = path.join(crate::constants::DB_DIR_NAME); + + // Get stats + let (total_chunks, total_files, model, lock_status) = if db_path.exists() { + let (model_name, dims) = read_model_metadata(&db_path); + + let lock = if crate::index::is_database_locked(&db_path) { + "conflicted" + } else { + "available" + }; + + if let Ok(store) = VectorStore::new(&db_path, dims) { + if let Ok(stats) = store.stats() { + ( + stats.total_chunks, + stats.total_files, + model_name, + lock.to_string(), + ) + } else { + (0, 0, model_name, lock.to_string()) + } + } else { + (0, 0, model_name, "readonly".to_string()) + } + } else { + (0, 0, "not indexed".to_string(), "unknown".to_string()) + }; + + // Stdio mode is single-repo-at-a-time CLI usage, not the live multi-repo + // federation this fan-out fix targets — a stats() failure here is out of + // scope for this fix (VectorStore::new/stats failing locally is a different + // shape than a store going down mid-request in a shared serve process). + repos_info.push(RepoInfo { + alias: alias.clone(), + project_path: path.display().to_string(), + database_path: db_path.display().to_string(), + total_chunks, + total_files, + model, + lock_status, + groups: project_groups.get(alias).cloned().unwrap_or_default(), + error: None, + }); + } + + let response = ListProjectsResponse { + repos: repos_info, + groups: config.groups_with_virtual_all(), + remote_projects: Self::remote_projects_listing(&config), + serve_active, + serve_url, + current_directory: current_dir.display().to_string(), + }; + + let json = serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string()); + Ok(CallToolResult::success(vec![ContentBlock::text(json)])) + } +} diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index b74bb293..3495b472 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -5,25 +5,53 @@ fn test_mcp_no_raw_stdout_calls() { // Verify that no raw print!/println! calls exist in the MCP module sources. // MCP communicates over stdout (JSON-RPC), so any stdout pollution breaks the protocol. // All informational output must go through info_print!/warn_print!/eprintln! (stderr). - let src = include_str!("mod.rs"); - let violations: Vec<(usize, &str)> = src - .lines() - .enumerate() - .filter(|(_, line)| { - let trimmed = line.trim_start(); - // Skip comments and lines that are part of the detection logic itself - if trimmed.starts_with("//") || trimmed.starts_with("\"") { - return false; - } - // Only flag lines that actually invoke print! or println! as a macro call - // (i.e. the identifier immediately followed by '!'), not lines discussing them - let call_println = line.contains("println!("); - let call_print = trimmed.starts_with("print!(") - || line.contains(" print!(") - || line.contains("\tprint!("); - let is_prefixed = line.contains("info_print!(") || line.contains("warn_print!("); - let is_detection_code = line.contains("line.contains("); - (call_println || call_print) && !is_prefixed && !is_detection_code + // Scans EVERY source file under src/mcp (todo #105 split the module), so a + // new per-tool file cannot silently escape the detector. + const MCP_SOURCES: &[(&str, &str)] = &[ + ("mod.rs", include_str!("mod.rs")), + ("search.rs", include_str!("search.rs")), + ("find.rs", include_str!("find.rs")), + ("explore.rs", include_str!("explore.rs")), + ("get_chunk.rs", include_str!("get_chunk.rs")), + ("status.rs", include_str!("status.rs")), + ("find_impact.rs", include_str!("find_impact.rs")), + ( + "find_impact_tracker.rs", + include_str!("find_impact_tracker.rs"), + ), + ("graph.rs", include_str!("graph.rs")), + ("literal_search.rs", include_str!("literal_search.rs")), + ( + "federation_helpers.rs", + include_str!("federation_helpers.rs"), + ), + ("helpers.rs", include_str!("helpers.rs")), + ("instructions.rs", include_str!("instructions.rs")), + ("responses.rs", include_str!("responses.rs")), + ("proxy.rs", include_str!("proxy.rs")), + ("runtime.rs", include_str!("runtime.rs")), + ("types.rs", include_str!("types.rs")), + ]; + let violations: Vec<(String, usize, &str)> = MCP_SOURCES + .iter() + .flat_map(|(file, src)| { + src.lines().enumerate().filter_map(move |(i, line)| { + let trimmed = line.trim_start(); + // Skip comments and lines that are part of the detection logic itself + if trimmed.starts_with("//") || trimmed.starts_with("\"") { + return None; + } + // Only flag lines that actually invoke print! or println! as a macro call + // (i.e. the identifier immediately followed by '!'), not lines discussing them + let call_println = line.contains("println!("); + let call_print = trimmed.starts_with("print!(") + || line.contains(" print!(") + || line.contains("\tprint!("); + let is_prefixed = line.contains("info_print!(") || line.contains("warn_print!("); + let is_detection_code = line.contains("line.contains("); + ((call_println || call_print) && !is_prefixed && !is_detection_code) + .then(|| ((*file).to_string(), i + 1, line)) + }) }) .collect(); @@ -32,12 +60,43 @@ fn test_mcp_no_raw_stdout_calls() { "MCP module has raw stdout calls that break the JSON-RPC protocol:\n{}", violations .iter() - .map(|(i, l)| format!(" line {}: {}", i + 1, l.trim())) + .map(|(f, i, l)| format!(" {}:{}: {}", f, i, l.trim())) .collect::>() .join("\n") ); } +// === Tool registration pin === +// +// Safety net for the mod.rs split: a `#[tool]` method that lands in an impl +// block the `#[tool_router]` macro does not scan is SILENTLY not registered. +// This asserts on the exact router the service wires up (`merged_tool_router`, +// the same expression both ctors and `#[tool_handler]` use) so every +// extraction stage must keep the 6-tool surface intact. + +#[test] +fn test_tool_registration_exposes_exactly_the_six_tools() { + let router = super::CodesearchService::merged_tool_router(); + let mut names: Vec = router + .list_all() + .into_iter() + .map(|t| t.name.to_string()) + .collect(); + names.sort(); + let expected: Vec<&str> = vec![ + "explore", + "find", + "find_impact", + "get_chunk", + "search", + "status", + ]; + assert_eq!( + names, expected, + "tools/list must expose exactly the consolidated 6-tool surface" + ); +} + #[cfg(windows)] #[test] fn test_mcp_filter_matches_absolute_path_under_project_root() { @@ -164,7 +223,7 @@ fn pick_filter_root_stdio_falls_back_to_project_path() { fn ns_item(path: &str) -> super::SearchResultItem { super::SearchResultItem { - chunk_id: 1, + chunk_id: Some(1), path: path.to_string(), start_line: 1, end_line: 2, @@ -334,7 +393,7 @@ fn test_low_confidence_response_serialization() { fn test_normal_response_omits_confidence_fields() { let response = super::SemanticSearchResponse { results: vec![super::SearchResultItem { - chunk_id: 1, + chunk_id: Some(1), path: "test.rs".to_string(), start_line: 1, end_line: 10, @@ -890,7 +949,7 @@ fn test_literal_search_result_item_omits_none_fields() { fn test_semantic_search_response_with_results() { let response = super::SemanticSearchResponse { results: vec![super::SearchResultItem { - chunk_id: 1, + chunk_id: Some(1), path: "test.rs".to_string(), start_line: 1, end_line: 10, @@ -1985,14 +2044,14 @@ fn note_store_failure_survives_a_short_alias_list() { #[test] fn index_status_summary_reports_building_before_anything_failed() { - let (status, message) = super::index_status_summary(3, 0, 0); + let (status, message) = super::index_status_summary(3, 0, 0, true); assert_eq!(status, "building"); assert!(!message.contains("failed"), "got: {message}"); } #[test] fn index_status_summary_reports_clean_ready_with_no_failures() { - let (status, message) = super::index_status_summary(3, 0, 500); + let (status, message) = super::index_status_summary(3, 0, 500, true); assert_eq!(status, "ready"); assert!(!message.contains("failed"), "got: {message}"); assert!(message.contains("3 repo(s)"), "got: {message}"); @@ -2002,7 +2061,7 @@ fn index_status_summary_reports_clean_ready_with_no_failures() { fn index_status_summary_surfaces_a_degraded_group_as_ready_with_a_count() { // This is the exact case that used to be indistinguishable from // "index still warming": some data is in, one store didn't answer. - let (status, message) = super::index_status_summary(3, 1, 500); + let (status, message) = super::index_status_summary(3, 1, 500, true); assert_eq!( status, "ready", "the two healthy stores must not be masked by the one that failed" @@ -2022,7 +2081,7 @@ fn index_status_summary_reports_error_when_every_store_failed() { // this fix, `total_chunks == 0` was checked first and this rendered as // "building" — byte-identical to "not indexed yet" — even though every // store actively failed. `failed_count >= total_repos` must win. - let (status, message) = super::index_status_summary(3, 3, 0); + let (status, message) = super::index_status_summary(3, 3, 0, true); assert_eq!( status, "error", "a group where every store failed must not read as merely 'still building'" @@ -2177,10 +2236,10 @@ fn qualify_empty_result_contradicts_a_not_found_diagnosis() { #[test] fn respond_with_items_carries_warnings_on_every_path() { - use rmcp::model::RawContent; + use rmcp::model::ContentBlock; let text = |r: Result| -> String { - match &r.unwrap().content[0].raw { - RawContent::Text(t) => t.text.clone(), + match &r.unwrap().content[0] { + ContentBlock::Text(t) => t.text.clone(), other => panic!("expected text content, got {other:?}"), } }; @@ -2215,6 +2274,128 @@ fn respond_with_items_carries_warnings_on_every_path() { assert_eq!(out, "No indexed chunks found for path."); } +#[test] +fn respond_with_items_noted_shapes_on_every_path() { + use rmcp::model::ContentBlock; + let text = |r: Result| -> String { + match &r.unwrap().content[0] { + ContentBlock::Text(t) => t.text.clone(), + other => panic!("expected text content, got {other:?}"), + } + }; + let warned = vec!["repo 'inriver' usage search failed: os error 22".to_string()]; + let note = Some("lexical text matching — use find_impact for precise references"); + + // Healthy, no note: byte-identical bare array (the delegated legacy path). + let out = text(super::respond_with_items_noted( + &[1u32, 2], + &[], + None, + || "unused".to_string(), + )); + assert_eq!( + out, "[1,2]", + "healthy no-note response must not change shape" + ); + + // Note only: results + note, no warnings key (absent, not null). + let out = text(super::respond_with_items_noted( + &[1u32, 2], + &[], + note, + || "unused".to_string(), + )); + let p: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!(p["results"][0], 1); + assert!(p["note"].as_str().unwrap().contains("find_impact")); + assert!(p.get("warnings").is_none(), "no null warnings key: {p}"); + + // Note + warnings: the channel still terminates, next to the note. + let out = text(super::respond_with_items_noted( + &[1u32], + &warned, + note, + || "unused".to_string(), + )); + let p: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!(p["note"].as_str().is_some(), "got: {p}"); + assert_eq!(p["warnings"][0], warned[0].as_str()); + + // Warnings only (note absent): identical to respond_with_items shape. + let out = text(super::respond_with_items_noted( + &[1u32], + &warned, + None, + || "unused".to_string(), + )); + let p: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!(p.get("note").is_none(), "got: {p}"); + assert!(p["warnings"][0].as_str().is_some(), "got: {p}"); + + // Empty + note: the note rides on the empty message — an empty lexical + // result is exactly where the SCIP upgrade path matters most. + let out = text(super::respond_with_items_noted( + &[0u32; 0], + &[], + note, + || "No usages found for 'Foo'.".to_string(), + )); + assert!(out.contains("No usages found for 'Foo'."), "got: {out}"); + assert!(out.contains("find_impact"), "got: {out}"); +} + +#[test] +fn rank_code_first_demotes_docs_without_reordering_code() { + let item = |path: &str, score: f32| super::ReferenceItem { + chunk_id: 0, + path: path.to_string(), + line: 1, + kind: "Block".to_string(), + signature: None, + score, + }; + // Score order deliberately NOT aligned with code/doc grouping: the + // highest-scoring hit is markdown. Stable sort must keep cs before ts + // (both code, 5.0 before 4.0) and md before AGENTS.md (both docs, + // 9.0 before 1.0), while every code item outranks every doc item. + let mut items = vec![ + item("docs/README.md", 9.0), + item("src/A.cs", 5.0), + item("src/b.ts", 4.0), + item("AGENTS.md", 1.0), + ]; + super::rank_code_first(&mut items); + let paths: Vec<&str> = items.iter().map(|i| i.path.as_str()).collect(); + assert_eq!( + paths, + ["src/A.cs", "src/b.ts", "docs/README.md", "AGENTS.md"], + "code first (score order kept), docs demoted as a block: {paths:?}" + ); +} + +#[test] +fn scip_usages_note_is_suppressed_without_backed_source_files() { + // Machine-independent half of the gate: no C#/TS source in the hits → + // no note, regardless of what indexers the host happens to have (the + // registry must not even be consulted). The positive branch is gated on + // helper availability and therefore structure-covered only, same + // reasoning as the csharp_helper_integration test. + let registry = std::sync::Arc::new(crate::symbols::SymbolIndexerRegistry::new()); + let item = |path: &str| super::ReferenceItem { + chunk_id: 0, + path: path.to_string(), + line: 1, + kind: "Block".to_string(), + signature: None, + score: 1.0, + }; + let markdown_only = vec![item("docs/notes.md"), item("src/lib.rs"), item("AGENTS.md")]; + assert!( + super::scip_usages_note(®istry, &markdown_only, "Foo").is_none(), + "a Rust/docs hit list must not advertise a SCIP upgrade path" + ); +} + #[test] fn ambiguous_chunk_payload_declares_an_incomplete_candidate_list() { // `candidate_projects` reads as exhaustive. When a store failed to @@ -2246,10 +2427,10 @@ fn ambiguous_chunk_payload_is_unchanged_when_every_store_answered() { #[test] fn respond_with_object_carries_warnings_without_disturbing_the_healthy_shape() { - use rmcp::model::RawContent; + use rmcp::model::ContentBlock; let text = |r: Result| -> String { - match &r.unwrap().content[0].raw { - RawContent::Text(t) => t.text.clone(), + match &r.unwrap().content[0] { + ContentBlock::Text(t) => t.text.clone(), other => panic!("expected text content, got {other:?}"), } }; @@ -2432,3 +2613,198 @@ fn test_grep_format_no_comment_when_plain() { let output = lines.join("\n"); assert!(!output.starts_with('#')); } + +// === shared-store second-open policy (stdio MCP vs standalone CLI) === + +#[test] +fn stdio_shared_stores_must_not_open_second_vector_store() { + // codesearch mcp --mode local: SharedStores is Some, serve_state is None. + // Old code gated only on serve_state and still called VectorStore::new / open_readonly. + assert!( + !super::allow_vector_store_second_open(true), + "stdio MCP with live shared_stores must not open a second LMDB VectorStore" + ); +} + +#[test] +fn standalone_cli_without_shared_stores_may_open_vector_store() { + assert!(super::allow_vector_store_second_open(false)); +} + +#[tokio::test] +async fn shared_store_failure_preserves_original_error_without_reopening_lmdb() { + let root = tempfile::tempdir().expect("tempdir"); + let db_path = root.path().join(".codesearch.db"); + let stores = + std::sync::Arc::new(crate::index::SharedStores::new(&db_path, 2).expect("shared stores")); + std::fs::write( + db_path.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .expect("metadata"); + let service = + super::CodesearchService::new_with_stores(Some(root.path().to_path_buf()), Some(stores)) + .expect("service"); + + let result: anyhow::Result<()> = service + .with_vector_store_read_for( + |_| Err(anyhow::anyhow!("sentinel shared-store read failure")), + None, + ) + .await; + let error = result.expect_err("shared-store failure must propagate"); + assert!( + format!("{error:#}").contains("sentinel shared-store read failure"), + "fallback must not replace the original error with a second-open error" + ); +} + +#[test] +fn prebuilt_index_health_requires_vector_and_full_text_data() { + let db_path = std::path::Path::new("/tmp/test-codesearch.db"); + assert!(super::validate_prebuilt_index_health(db_path, 10, true, 10, false).is_ok()); + + for (chunks, indexed, fts_documents, partial) in [ + (0, true, 10, false), + (10, false, 10, false), + (10, true, 0, false), + (10, true, 10, true), + ] { + let error = + super::validate_prebuilt_index_health(db_path, chunks, indexed, fts_documents, partial) + .expect_err("incomplete prebuilt index must fail"); + assert!(error.to_string().contains("Run `codesearch index`")); + } +} + +#[tokio::test] +async fn require_ready_reads_live_stores_and_rejects_partial_metadata() { + let root = tempfile::tempdir().expect("tempdir"); + let db_path = root.path().join(".codesearch.db"); + let stores = crate::index::SharedStores::new(&db_path, 2).expect("shared stores"); + { + let mut vector_store = stores.vector_store.write().await; + let chunk = crate::chunker::Chunk::new( + "fn ready() {}".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/lib.rs".to_string(), + ); + vector_store + .insert_chunks(vec![crate::embed::EmbeddedChunk::new( + chunk, + vec![0.0, 1.0], + )]) + .expect("insert vector chunk"); + vector_store.build_index().expect("build vector index"); + } + { + let mut fts_store = stores.fts_store.write().await; + fts_store + .add_chunk(0, "fn ready() {}", "src/lib.rs", None, "Function") + .expect("insert FTS document"); + fts_store.commit().expect("commit FTS document"); + } + std::fs::write( + db_path.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"partial":false}"#, + ) + .expect("complete metadata"); + + super::require_prebuilt_index_ready(&stores, &db_path) + .await + .expect("complete live stores must be ready"); + + std::fs::write( + db_path.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"partial":true}"#, + ) + .expect("partial metadata"); + let error = super::require_prebuilt_index_ready(&stores, &db_path) + .await + .expect_err("partial index must not be ready"); + assert!(error.to_string().contains("partial=true")); + + std::fs::write( + db_path.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2}"#, + ) + .expect("metadata without readiness marker"); + let error = super::require_prebuilt_index_ready(&stores, &db_path) + .await + .expect_err("missing partial marker must not be ready"); + assert!(error + .to_string() + .contains("missing the partial readiness marker")); + + std::fs::write( + db_path.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"partial":"false"}"#, + ) + .expect("malformed metadata"); + let error = super::require_prebuilt_index_ready(&stores, &db_path) + .await + .expect_err("non-boolean partial marker must not be ready"); + assert!(error.to_string().contains("partial must be a boolean")); +} + +#[test] +fn local_startup_flags_fail_in_modes_that_would_ignore_them() { + for mode in [super::McpMode::Auto, super::McpMode::Client] { + assert!(super::validate_local_startup_flags(mode, true, false).is_err()); + assert!(super::validate_local_startup_flags(mode, false, true).is_err()); + } + assert!(super::validate_local_startup_flags(super::McpMode::Local, true, true).is_ok()); +} + +#[test] +fn readonly_or_require_ready_never_creates_a_missing_index() { + assert!(super::may_create_missing_index(true, false, false)); + assert!(!super::may_create_missing_index(false, false, false)); + assert!(!super::may_create_missing_index(true, true, false)); + assert!(!super::may_create_missing_index(true, false, true)); +} + +/// Chunks without a built vector index are NOT searchable, and must not read as +/// `ready`. Regression guard for a rebuild window (and the mixed-state hub the +/// model migration exposed): `total_chunks > 0` used to be sufficient for +/// "ready", so `status` said "Index is ready for searching" while every search +/// failed with "Index not built. Call build_index() after inserting chunks." +#[test] +fn index_status_summary_reports_building_when_chunks_exist_but_graph_is_not_built() { + let (status, message) = super::index_status_summary(3, 0, 500, false); + assert_eq!( + status, "building", + "chunks without a built vector index are not searchable" + ); + assert!(message.contains("not built"), "got: {message}"); + assert!( + message.contains("3 repo(s)"), + "message must still name the scope, got: {message}" + ); + // A stats() failure on some stores must NOT be misread as "not built". + let (status, _) = super::index_status_summary(3, 1, 500, true); + assert_eq!( + status, "ready", + "a failed store is surfaced as degraded-ready" + ); +} + +/// Single-store counterpart: the same `total_chunks > 0` shortcut reported +/// `ready` for a store whose graph was never built (`indexed == false`). +#[test] +fn single_index_status_requires_a_built_graph_to_report_ready() { + let (status, message) = super::single_index_status(403, false); + assert_eq!( + status, "building", + "403 chunks with no built graph are not searchable" + ); + assert!(message.contains("not built"), "got: {message}"); + + let (status, _) = super::single_index_status(403, true); + assert_eq!(status, "ready"); + + let (status, _) = super::single_index_status(0, false); + assert_eq!(status, "building"); +} diff --git a/src/mcp/types.rs b/src/mcp/types.rs index 56f6bc5a..e113eff8 100644 --- a/src/mcp/types.rs +++ b/src/mcp/types.rs @@ -139,10 +139,13 @@ pub struct StatusRequest { /// Input variants: /// - By name: `{ "symbol_name": "FieldDefinition.Validate", "project": "myrepo" }` /// - By position: `{ "file": "src/Validation/FieldDefinition.cs", "line": 42, "project": "myrepo" }` +/// - By exact key (explicit selection after an ambiguous answer): +/// `{ "symbol_key": "csharp . . . FieldDefinition#Validate().", "project": "myrepo" }` #[derive(Debug, Deserialize, Serialize, JsonSchema)] pub struct FindImpactRequest { /// Symbol name to look up (e.g. `"FieldDefinition.Validate"`). - /// Used when you know the name. Mutually exclusive with `file`+`line`. + /// Mutually exclusive with `symbol_key`; if both a name and + /// file+line arrive, the name wins (documented precedence). pub symbol_name: Option, /// File path for position-based lookup (relative to project root or absolute). @@ -153,8 +156,19 @@ pub struct FindImpactRequest { /// Must be combined with `file`. pub line: Option, - /// Language filter (e.g. `"csharp"`). If omitted, auto-detects from file extension - /// or searches all installed language adapters. + /// Exact canonical SCIP symbol key — the explicit selection after an + /// ambiguous answer listed candidates (or any full key the caller + /// already has). Looked up verbatim: no fuzzy fallback, no silent + /// overload picking. Mutually exclusive with `symbol_name` and + /// `file`+`line`. + #[serde(default)] + pub symbol_key: Option, + + /// Language filter (e.g. `"csharp"`). If omitted: position lookups + /// auto-detect it from the file extension; with no or exactly one + /// installed helper the pick is deterministic, and with several + /// installed the answer asks you to name one instead of picking + /// silently. pub language: Option, /// Route to a specific project (requires `codesearch serve`). @@ -268,7 +282,17 @@ pub struct SimilarChunksRequest { /// Search result item — returned by semantic search #[derive(Debug, Serialize, Deserialize)] pub struct SearchResultItem { - pub chunk_id: u32, + /// Store-assigned chunk id. `Some` for semantic hits (where the store + /// returned a real id); **`None` for literal-mode hits** — literal results + /// carry no chunk id and the field is omitted from the JSON entirely. + /// + /// Never fabricate an id (e.g. `unwrap_or(0)`) for absent values: a + /// caller can combine a rendered `0` with the item's `source` into a + /// bogus `chunk_ref` (`"/:0"`) that `get_chunk` will + /// silently resolve to an *unrelated* chunk. An absent id must render + /// as an absent field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chunk_id: Option, pub path: String, pub start_line: usize, pub end_line: usize, diff --git a/src/serve/mod.rs b/src/serve/mod.rs index f8ee7df8..8bca5806 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -3,8 +3,8 @@ //! Binds on `{host}:{port}` (default `127.0.0.1:39725`) and serves: //! - `GET /health` → JSON health check //! - `POST /repos` → register + index + warmup a new repo -//! - `DELETE /repos/:alias` → stop FSW + evict + unregister + delete DB -//! - `POST /repos/:alias/reindex` → trigger incremental or force reindex +//! - `DELETE /repos/{alias}` → stop FSW + evict + unregister + delete DB +//! - `POST /repos/{alias}/reindex` → trigger incremental or force reindex //! - MCP streamable HTTP at `/mcp` via rmcp tower service //! //! Holds a `DashMap>` keyed by repo alias. @@ -37,11 +37,11 @@ use crate::cache::safe_canonicalize; use crate::constants::{ ALLOWED_HOSTS_ENV, ALLOWED_ROOTS_ENV, CHUNK_PATH, CSHARP_PREWARM_ENABLED_ENV, CSHARP_PREWARM_MAX_SYMBOLS, CSHARP_SCIP_CONCURRENCY_DEFAULT, CSHARP_SCIP_CONCURRENCY_ENV, - DB_DIR_NAME, DEFAULT_SERVE_PORT, DISABLE_HOST_VALIDATION_ENV, EXPLORE_PATH, FIND_PATH, - HEALTHZ_PATH, HEALTH_PATH, LANG_CSHARP, LANG_TYPESCRIPT, MAX_INDEXING_SECS, - MAX_INDEXING_SECS_ENV, MCP_ENDPOINT_PATH, PERSIST_DEBOUNCE_SECS, REAPER_INTERVAL_SECS, - REMOTES_PATH, REPO_IDLE_TIMEOUT_ENV, REPO_IDLE_TIMEOUT_SECS, SEARCH_PATH, SERVE_API_KEY_ENV, - SERVE_PORT_ENV, STATUS_PATH, + DB_DIR_NAME, DEFAULT_SERVE_PORT, DISABLE_HOST_VALIDATION_ENV, EXPLORE_PATH, FIND_IMPACT_PATH, + FIND_PATH, HEALTHZ_PATH, HEALTH_PATH, INDEXING_PATH, LANG_CSHARP, LANG_TYPESCRIPT, + MAX_INDEXING_SECS, MAX_INDEXING_SECS_ENV, MCP_ENDPOINT_PATH, PERSIST_DEBOUNCE_SECS, + REAPER_INTERVAL_SECS, REMOTES_PATH, REPO_IDLE_TIMEOUT_ENV, REPO_IDLE_TIMEOUT_SECS, SEARCH_PATH, + SERVE_API_KEY_ENV, SERVE_PORT_ENV, STATUS_PATH, }; use crate::db_discovery::repos::{config_dir, ReposConfig}; use crate::index::{ @@ -187,6 +187,22 @@ pub(crate) struct ServeState { /// Repo alias → timestamp of last query that touched this repo. /// Used by the idle-reaper to evict repos after `REPO_IDLE_TIMEOUT_SECS`. last_access: DashMap, + /// Repo alias → cold-open single-flight lock (see [`Self::open_lock`]). + /// + /// A cold open (fast-path miss → `try_open_stores` → insert) must never run + /// concurrently with another cold open of the SAME alias: the second LMDB + /// open trips the double-open guard and caches `RepoState::Conflicted`, + /// which the Conflicted self-heal can then never cure while the first + /// opener's env is still alive — the winner of the race holds the env from + /// `try_open_stores` until its insert, and a request stuck in between (e.g. + /// a long HNSW build) wedges the repo for the process lifetime + /// (todo #131, 2026-09-08 incident). Both cold-open entry points + /// (`get_or_open_stores`, `warmup_repo`) hold this lock across their slow + /// path and RE-CHECK the fast path after acquiring it, so the race loser + /// waits and then hits the winner's cache entry. The `Arc` indirection + /// keeps the DashMap shard guard short-lived — never held across the lock + /// await. + open_locks: DashMap>>, /// Repo alias → `JoinHandle` of its background file-system-watcher (FSW) task. /// /// The FSW task holds its own clones of `Arc` and @@ -214,6 +230,18 @@ pub(crate) struct ServeState { /// drop first. The token is stored alongside the handle so `remove_repo` /// can cancel regardless of the repo's `RepoState` variant. index_tasks: DashMap, CancellationToken)>, + /// Aliases detected with LMDB storage-format corruption (e.g. + /// `MDB_BAD_VALSIZE` after a storage-layer major upgrade such as + /// arroy 0.5→0.8 / heed 0.20→0.22), queued for a wipe + full force + /// reindex. Processed strictly one at a time by the recovery worker — + /// each rebuild runs a full CPU-bound embed pass, so parallel + /// recoveries would thrash the machine. See + /// [`Self::enqueue_format_recovery`] / [`Self::recover_repo_format`]. + format_recovery_queue: std::sync::Mutex>, + /// Guarantees at most one recovery worker is alive. A worker that finds + /// the queue empty flips this back to `false` under the queue lock, so an + /// enqueue racing the worker's exit re-spawns cleanly (no lost wake-up). + format_recovery_worker_started: std::sync::atomic::AtomicBool, /// Loaded repos config (alias → path). config: std::sync::RwLock, /// Last observed mtime of the repos config file. @@ -253,12 +281,33 @@ pub(crate) struct ServeState { /// `find_impact` to reuse helper-detection cache instead of creating fresh /// instances per request. symbol_registry: Arc, - /// Shared embedding service — used by MCP sessions AND the REST handlers so - /// the ONNX embedding model is loaded ONCE per serve instance (lazily, on - /// the first semantic query) and reused across all requests. Without this, - /// per-request `CodesearchService` construction (REST handlers) would reload - /// the model on every call (~100ms–2s). Mirrors the `symbol_registry` pattern. - embedding_service: Arc>>, + /// Shared, per-model embedding-service pool — used by MCP sessions AND the + /// REST handlers so each ONNX embedding model is loaded ONCE per serve + /// instance (lazily, on the first semantic query) and reused across all + /// requests. Without this, per-request `CodesearchService` construction + /// (REST handlers) would reload the model on every call (~100ms–2s). + /// + /// A pool rather than a single service because serve is multi-repo and + /// indexes may be built with different models: every query must be embedded + /// with the model of the repo it targets. Mirrors the `symbol_registry` + /// pattern. + embedding_pool: Arc, + /// Serve-wide default embedding model for newly created indexes + /// (`codesearch serve --model `), or `None` for the built-in default. + /// + /// This never overrides an index that already records its own model, and it + /// is deliberately NOT the query fallback for an index that records none: + /// a legacy index with no `model_short_name` is queried with the built-in + /// default and reported with a warning (see + /// `CodesearchService::resolve_query_model`). Applying this flag there would + /// break a working legacy repo the moment an operator set it. The default + /// applies only when `POST /repos` creates a brand-new index without an + /// explicit `model`, and to the scope-free status summary. + default_model: Option, + /// Aliases for which the unrecorded-model query warning has already been + /// emitted, so a long-running serve logs it once per repo instead of once + /// per query. See [`Self::mark_legacy_model_warned`]. + legacy_model_warned: DashMap, /// Per-repo total tool call count. tool_call_counts: DashMap, /// Per-repo C# symbol index status (cached, updated on rebuild/detect). @@ -324,8 +373,11 @@ impl ServeState { Self { repos: DashMap::new(), last_access: DashMap::new(), + open_locks: DashMap::new(), fsw_tasks: DashMap::new(), index_tasks: DashMap::new(), + format_recovery_queue: std::sync::Mutex::new(std::collections::VecDeque::new()), + format_recovery_worker_started: std::sync::atomic::AtomicBool::new(false), config: std::sync::RwLock::new(config), config_mtime: std::sync::RwLock::new(None), config_path_override, @@ -337,7 +389,11 @@ impl ServeState { total_sessions: AtomicU64::new(0), sysinfo_system: std::sync::Mutex::new(sys), symbol_registry: Arc::new(SymbolIndexerRegistry::new()), - embedding_service: Arc::new(std::sync::Mutex::new(None)), + embedding_pool: Arc::new(crate::embed::EmbeddingServicePool::new( + crate::constants::get_global_models_cache_dir().ok(), + )), + default_model: None, + legacy_model_warned: DashMap::new(), tool_call_counts: DashMap::new(), csharp_index_status: Arc::new(DashMap::new()), csharp_index_error: Arc::new(DashMap::new()), @@ -349,6 +405,79 @@ impl ServeState { } } + /// Per-alias cold-open single-flight lock. Cloned out of the map so the + /// DashMap shard guard is never held across the lock's `.await`. + fn open_lock(&self, alias: &str) -> Arc> { + self.open_locks + .entry(alias.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + + /// Fast-path lookup: `Some(result)` when `alias` already has opened stores + /// in the cache, `None` when a cold open is needed. Shared by the pre-lock + /// fast path and the re-check after the single-flight acquire — identical + /// semantics both times, including the Warm → Write transition on touch. + fn try_cached_stores( + &self, + alias: &str, + touch: bool, + ) -> Option, String>> { + let entry = self.repos.get(alias)?; + if touch { + self.touch_access(alias); + } + Some(match entry.value() { + RepoState::Write { stores, .. } | RepoState::Readonly { stores } => Ok(stores.clone()), + RepoState::Warm { stores } => { + // Lazy FSW start: transition Warm → Write only on real query access. + // Fan-out/candidate-detection callers pass touch=false and must not + // trigger Warm → Write or start FSW. + let stores = stores.clone(); + if !touch { + return Some(Ok(stores)); + } + drop(entry); // release DashMap read guard before mutation + + // Only one caller should do the transition; use a compare-and-swap pattern. + // Check if someone else already transitioned it. + if let Some(mut mut_entry) = self.repos.get_mut(alias) { + if let RepoState::Write { stores, .. } = mut_entry.value() { + return Some(Ok(stores.clone())); + } + if let RepoState::Warm { stores } = mut_entry.value() { + let stores = stores.clone(); + let path = { + let config = match self.config.read() { + Ok(c) => c, + Err(e) => return Some(Err(format!("Mutex poisoned: {}", e))), + }; + match config.resolve(alias) { + Some(p) => p, + None => { + return Some(Err(format!("Unknown alias '{}'", alias))); + } + } + }; + + // Start FSW in background for this repo + self.spawn_fsw_for_warm(alias, &path, stores.clone(), &mut mut_entry); + return Some(Ok(stores)); + } + // Someone else transitioned it already + if let RepoState::Readonly { stores } = mut_entry.value() { + return Some(Ok(stores.clone())); + } + if let RepoState::Conflicted = mut_entry.value() { + return Some(Err(Self::conflicted_msg(alias))); + } + } + Ok(stores) + } + RepoState::Conflicted => Err(Self::conflicted_msg(alias)), + }) + } + /// Return a clone of the shared symbol indexer registry Arc. /// Used by MCP sessions (CodesearchService::new_for_serve) and /// HTTP reindex handler (trigger_symbol_rebuild) to reuse @@ -357,14 +486,56 @@ impl ServeState { Arc::clone(&self.symbol_registry) } - /// Return a clone of the shared embedding-service Arc. - /// Shared across MCP sessions AND REST handlers so the ONNX model is loaded + /// Return a clone of the shared, per-model embedding-service pool. + /// Shared across MCP sessions AND REST handlers so each ONNX model is loaded /// once per serve instance (lazily on first semantic query) instead of being /// reloaded per request/session. - pub(crate) fn embedding_service( - &self, - ) -> Arc>> { - Arc::clone(&self.embedding_service) + pub(crate) fn embedding_pool(&self) -> Arc { + Arc::clone(&self.embedding_pool) + } + + /// Attach the serve-wide default embedding model (`codesearch serve --model`). + /// + /// Set once at startup, before the state is shared. `None` leaves the + /// built-in default in place. + pub(crate) fn with_default_model(mut self, model: Option) -> Self { + self.default_model = model; + self + } + + /// The serve-wide default embedding model for newly created indexes, or + /// `None` for the built-in default. See [`Self::with_default_model`]. + pub(crate) fn default_model(&self) -> Option { + self.default_model + } + + /// Resolve the embedding model an alias's index was built with. + /// + /// Returns `None` when the alias is unknown or its index has no + /// `model_short_name` (unindexed / legacy), so callers can fall back to + /// [`crate::embed::ModelType::default`]. This is the read side of the + /// per-repo model contract: a query against `alias` MUST be embedded with + /// the model returned here, or the vector search fails with a dimension + /// mismatch (768-dim EmbeddingGemma index, 384-dim default query) or + /// silently compares incomparable vector spaces. + pub(crate) fn model_for_alias(&self, alias: &str) -> Option { + let cfg = self.config_snapshot(); + let project_path = cfg.resolve(alias)?; + crate::embed::ModelType::from_index_metadata(&project_path.join(DB_DIR_NAME)) + } + + /// Record that `alias` was queried with the built-in default because its + /// index records no embedding model, returning `true` on the first call for + /// that alias. + /// + /// An unrecorded model is unknowable, so the fallback warning is logged once + /// per repo per serve lifetime rather than on every query — a busy hub would + /// otherwise flood the log with the same line. The caller-facing response + /// warning is not deduped: an agent should see the assumption on each answer. + pub(crate) fn mark_legacy_model_warned(&self, alias: &str) -> bool { + self.legacy_model_warned + .insert(alias.to_string(), ()) + .is_none() } /// Return the instant when serve started, used to compute uptime. @@ -456,6 +627,182 @@ impl ServeState { self.active_reindexes.remove(alias); } + /// True iff an error chain indicates LMDB storage-format corruption — + /// data written by an older storage-major (arroy/heed) that the current + /// one refuses to read — rather than a transient or unrelated failure. + fn is_lmdb_format_corruption(msg: &str) -> bool { + let m = msg.to_ascii_lowercase(); + m.contains("mdb_bad_valsize") + || m.contains("unsupported size of key") + || m.contains("wrong dupfixed size") + } + + /// Queue `alias` for a sequential wipe + force reindex after LMDB format + /// corruption was detected. Deduplicates; spawns the single recovery + /// worker on the first enqueue. + fn enqueue_format_recovery(self: &Arc, alias: &str) { + { + let mut queue = self + .format_recovery_queue + .lock() + .expect("format_recovery_queue lock poisoned"); + if queue.iter().any(|a| a == alias) { + return; + } + queue.push_back(alias.to_string()); + } + // Swap AFTER the push so the worker-exit path (which flips the flag + // back to `false` while still holding the queue lock) can never race + // us into a lost wake-up: either we observe `true` and the live + // worker picks up the fresh entry, or we flip `false→true` and spawn. + if !self + .format_recovery_worker_started + .swap(true, std::sync::atomic::Ordering::AcqRel) + { + let state = Arc::clone(self); + tokio::spawn(state.format_recovery_worker()); + } + } + + /// Pops queued aliases and recovers them ONE AT A TIME until the queue + /// runs dry, then exits (a later enqueue restarts a worker). + async fn format_recovery_worker(self: Arc) { + loop { + let alias = { + let mut queue = self + .format_recovery_queue + .lock() + .expect("format_recovery_queue lock poisoned"); + match queue.pop_front() { + Some(a) => a, + None => { + // Flip the flag while still holding the queue lock so + // a concurrent enqueue cannot interleave between the + // empty pop and the flag reset (lost wake-up). + self.format_recovery_worker_started + .store(false, std::sync::atomic::Ordering::Release); + return; + } + } + }; + if let Err(e) = self.recover_repo_format(&alias).await { + tracing::error!("🔧 Format recovery failed for '{}': {}", alias, e); + } + } + } + + /// Wipe + force reindex one repo whose on-disk storage was written by an + /// older storage-major. Mirrors `remove_repo`'s eviction sequence (stop + /// FSW → evict → await watcher/index shutdowns) but keeps the alias + /// registered; then deletes the DB directory (bounded retry for transient + /// Windows lock holders) and reuses the TUI force-reindex machinery — its + /// `try_open_stores` path recreates fresh stores when the directory is + /// gone, so the rebuild lands on the new arroy/heed formats. + async fn recover_repo_format(self: &Arc, alias: &str) -> Result<(), String> { + let project_path = { + let config = self + .config + .read() + .map_err(|_| "config lock poisoned".to_string())?; + if config.repo_read_only.get(alias) == Some(&true) { + return Err(format!( + "'{}' is marked read-only; rebuild its index on the owning writer", + alias + )); + } + config + .resolve(alias) + .ok_or_else(|| format!("unknown alias '{}'", alias))? + }; + let db_path = project_path.join(DB_DIR_NAME); + + // Evict in-memory holders so the LMDB env closes before the delete + // (Windows refuses to delete mmap'd files). Same order as remove_repo. + { + let _stores = self.stop_fsw(alias); + } + self.repos.remove(alias); + self.last_access.remove(alias); + self.await_fsw_shutdown(alias).await; + self.await_index_task(alias).await; + + let deadline = + Instant::now() + Duration::from_secs(crate::constants::DB_DELETE_RETRY_BUDGET_SECS); + let mut backoff_ms = crate::constants::DB_DELETE_RETRY_INITIAL_MS; + loop { + match std::fs::remove_dir_all(&db_path) { + Ok(()) => break, + Err(e) if e.kind() == std::io::ErrorKind::NotFound || !db_path.exists() => break, + Err(e) if Self::is_db_locked_error(&e) && Instant::now() < deadline => { + tracing::debug!( + "Format recovery: DB dir for '{}' still locked, retrying: {}", + alias, + e + ); + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; + backoff_ms = (backoff_ms * 2).min(2_000); + } + Err(e) => { + return Err(format!( + "could not wipe {} after corruption: {}", + db_path.display(), + e + )) + } + } + } + tracing::info!( + "🔧 Format recovery: wiped stale-format DB dir for '{}' — rebuilding", + alias + ); + + match tui::spawn_force_reindex(alias.to_string(), self) { + tui::ReindexLaunch::Started => { + // Sequential guarantee: wait until this alias stops indexing + // before the worker loop picks the next one. Poll — the + // active-reindexes entry can go stale (MAX_INDEXING_SECS) on + // very long rebuilds, so cap generously and surface a timeout + // rather than hanging the whole recovery queue. + let cap = self.indexing_timeout() * 8; + let started = Instant::now(); + while self.is_indexing(alias) { + if started.elapsed() >= cap { + return Err(format!( + "rebuild for '{}' exceeded the {}s recovery cap", + alias, + cap.as_secs() + )); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + tracing::info!("🔧 Format recovery: rebuild complete for '{}'", alias); + Ok(()) + } + tui::ReindexLaunch::AlreadyRunning => { + // A rebuild is already in flight for this alias; wait it out. + // If it was a plain rebuild it may fail on the corrupt dir + // again — the next rebuild trigger re-detects and re-queues. + let cap = self.indexing_timeout() * 8; + let started = Instant::now(); + while self.is_indexing(alias) { + if started.elapsed() >= cap { + return Err(format!( + "in-flight rebuild for '{}' exceeded the {}s recovery cap", + alias, + cap.as_secs() + )); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + Ok(()) + } + tui::ReindexLaunch::Failed => Err(format!( + "could not start the recovery rebuild for '{}' (see log)", + alias + )), + } + } + /// Returns `true` if `alias` is currently (non-stale) indexing. /// /// Stale entries — those older than [`MAX_INDEXING_SECS`] — are lazily @@ -1282,7 +1629,7 @@ impl ServeState { /// Remove a repo: stop FSW, evict from memory, unregister from config, delete DB. /// - /// This is the shared logic used by both the HTTP `DELETE /repos/:alias` handler + /// This is the shared logic used by both the HTTP `DELETE /repos/{alias}` handler /// and the TUI confirmation flow. pub(crate) async fn remove_repo(&self, alias: &str) -> Result { // 1. Resolve project path from config @@ -1419,9 +1766,50 @@ impl ServeState { alias, msg ); - tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; - backoff_ms = - (backoff_ms * 2).min(crate::constants::DB_DELETE_RETRY_BACKOFF_CAP_MS); + // A lock-class failure with the holders still IN THIS + // PROCESS is the common transient case (an in-flight + // search holding an `Arc` clone, a + // `spawn_blocking` embed pass). Blind backoff burns + // attempts against a directory that cannot possibly + // delete yet; instead wait on the registry — the single + // source of truth for "can this process delete the dir + // right now" — until every in-process env under the DB + // dir is released, then retry immediately. Only when + // the registry is ALREADY empty (the holder is + // external: another process, AV scanner) fall back to + // the exponential backoff above. + let holders = crate::lmdb_registry::open_holders_under(&db_path); + if holders.is_empty() { + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + backoff_ms = (backoff_ms * 2) + .min(crate::constants::DB_DELETE_RETRY_BACKOFF_CAP_MS); + } else { + tracing::debug!( + "DB delete for '{}': waiting for {} in-process LMDB holder(s) \ + to release: {:?}", + alias, + holders.len(), + holders + ); + let remaining = Self::await_lmdb_release(&db_path, deadline).await; + if remaining.is_empty() { + tracing::debug!( + "DB delete for '{}': in-process holders released; \ + retrying immediately", + alias + ); + } else { + // Budget expired with holders still present — + // the loop's deadline check breaks on the next + // iteration's error arm. + tracing::debug!( + "DB delete for '{}': budget expired with in-process \ + holder(s) still present: {:?}", + alias, + remaining + ); + } + } } } } @@ -1468,9 +1856,28 @@ impl ServeState { None } + /// Drop cached C# symbol-index status/error for `alias`. + /// + /// `repo_statuses_lightweight()` prefers these cached entries over its + /// on-disk probe, so a cached `Error` outlives the repo itself: a closed + /// repo has no watcher left to retry a rebuild or emit `Succeeded`, and + /// the red `C#!` it causes in the TUI freezes forever (observed on a repo + /// whose rebuild lost a one-shot LMDB double-open race days earlier). + /// Remove the entries letting the probe (helper available + index + /// exists → Ready) restore the on-disk truth. Called from idle eviction + /// and `close_repo` (force-reindex reopen). `remove_repo` deliberately + /// does NOT call this: once the alias is unregistered the entries are + /// display-unreachable (`repo_statuses_lightweight` iterates registered + /// repos only), so a clear there would be dead code. + fn clear_csharp_index_state(&self, alias: &str) { + self.csharp_index_status.remove(alias); + self.csharp_index_error.remove(alias); + } + /// Remove a repo from the DashMap, dropping its stores and releasing /// LMDB file handles. Used before force-reindex reopen. fn close_repo(&self, alias: &str) { + self.clear_csharp_index_state(alias); if self.repos.remove(alias).is_some() { tracing::info!( "Closed repo '{}' (dropped stores, released LMDB handles)", @@ -1608,6 +2015,35 @@ impl ServeState { || msg.contains("busy") } + /// Wait until no in-process LMDB env remains open under `db_path`, polling + /// [`crate::lmdb_registry::open_holders_under`] every + /// [`crate::constants::DB_DELETE_ENV_RELEASE_POLL_MS`]. + /// + /// Returns the holder descriptions still open when `deadline` was reached + /// — an empty `Vec` means every in-process holder released in time and the + /// directory is (as far as THIS process is concerned) immediately + /// deletable again. Holders owned by OTHER processes are invisible to the + /// registry by design; callers cover that case with plain backoff. + /// + /// The registry is the single source of truth this waits on: every holder + /// shape that can keep the LMDB mmap open on Windows — an outer + /// `Arc` clone held by an in-flight search, an inner + /// `Arc>` captured by a `spawn_blocking` embed pass, a + /// `SCIP(...)` env in a `scip/` subdirectory — keeps its `TrackedEnv` + /// (and therefore its registry slot) alive until it is truly dropped. + async fn await_lmdb_release(db_path: &Path, deadline: std::time::Instant) -> Vec { + loop { + let holders = crate::lmdb_registry::open_holders_under(db_path); + if holders.is_empty() || std::time::Instant::now() >= deadline { + return holders; + } + tokio::time::sleep(std::time::Duration::from_millis( + crate::constants::DB_DELETE_ENV_RELEASE_POLL_MS, + )) + .await; + } + } + /// Best-effort delete of an orphaned `.codesearch.db` directory, called /// from a background indexing task's post-build guard when its alias was /// removed (or cancelled) mid-build. The caller MUST drop its own @@ -1786,6 +2222,21 @@ impl ServeState { } } + // Single-flight per alias (see get_or_open_stores): a warmup racing a + // first query — or another warmup — must not reach try_open_stores + // twice, or the loser trips the LMDB double-open guard and the repo + // wedges as an incurable Conflicted (todo #131). + let open_lock = self.open_lock(alias); + let _open_guard = open_lock.lock().await; + if let Some(entry) = self.repos.get(alias) { + match entry.value() { + RepoState::Write { .. } | RepoState::Warm { .. } | RepoState::Readonly { .. } => { + return Ok(()); + } + RepoState::Conflicted => return Err(Self::conflicted_msg(alias)), + } + } + let (path, force_readonly) = { let config = self .config @@ -1801,7 +2252,7 @@ impl ServeState { let db_path = path.join(DB_DIR_NAME); // Open stores: existence check + write/readonly/conflicted logic. - let stores = match self.try_open_stores(alias, &db_path, false, force_readonly)? { + let stores = match self.try_open_stores(alias, &db_path, false, force_readonly, None)? { OpenedStores::Readonly(stores) => { // Already registered as Readonly by try_open_stores. // @@ -1972,58 +2423,19 @@ impl ServeState { } // Fast path: already opened - if let Some(entry) = self.repos.get(alias) { - if touch { - self.touch_access(alias); - } - return match entry.value() { - RepoState::Write { stores, .. } | RepoState::Readonly { stores } => { - Ok(stores.clone()) - } - RepoState::Warm { stores } => { - // Lazy FSW start: transition Warm → Write only on real query access. - // Fan-out/candidate-detection callers pass touch=false and must not - // trigger Warm → Write or start FSW. - let stores = stores.clone(); - if !touch { - return Ok(stores); - } - drop(entry); // release DashMap read guard before mutation - - // Only one caller should do the transition; use a compare-and-swap pattern. - // Check if someone else already transitioned it. - if let Some(mut mut_entry) = self.repos.get_mut(alias) { - if let RepoState::Write { stores, .. } = mut_entry.value() { - return Ok(stores.clone()); - } - if let RepoState::Warm { stores } = mut_entry.value() { - let stores = stores.clone(); - let path = { - let config = self - .config - .read() - .map_err(|e| format!("Mutex poisoned: {}", e))?; - config - .resolve(alias) - .ok_or_else(|| format!("Unknown alias '{}'", alias))? - }; + if let Some(result) = self.try_cached_stores(alias, touch) { + return result; + } - // Start FSW in background for this repo - self.spawn_fsw_for_warm(alias, &path, stores.clone(), &mut mut_entry); - return Ok(stores); - } - // Someone else transitioned it already - if let RepoState::Readonly { stores } = mut_entry.value() { - return Ok(stores.clone()); - } - if let RepoState::Conflicted = mut_entry.value() { - return Err(Self::conflicted_msg(alias)); - } - } - Ok(stores) - } - RepoState::Conflicted => Err(Self::conflicted_msg(alias)), - }; + // Single-flight per alias: wait for any in-flight cold open of this + // repo, then re-check the cache. Without this, two concurrent cold + // opens both reach try_open_stores; the second trips the LMDB + // double-open guard and caches Conflicted — incurable while the first + // opener holds its env (todo #131). + let open_lock = self.open_lock(alias); + let _open_guard = open_lock.lock().await; + if let Some(result) = self.try_cached_stores(alias, touch) { + return result; } // Slow path: need to open @@ -2042,7 +2454,7 @@ impl ServeState { let db_path = path.join(DB_DIR_NAME); // Open stores: existence check + write/readonly/conflicted logic. - let stores = match self.try_open_stores(alias, &db_path, false, force_readonly)? { + let stores = match self.try_open_stores(alias, &db_path, false, force_readonly, None)? { OpenedStores::Readonly(s) => { // Already registered as Readonly; touch and return. self.touch_access(alias); @@ -2288,12 +2700,20 @@ impl ServeState { /// /// `allow_create=false`: warmup / incremental reindex path — fails if DB is missing. /// `allow_create=true`: force-reindex / add-repo path — creates fresh DB if missing. + /// + /// `dimension_override` forces the embeddings dimension (e.g. a model + /// override on `POST /repos`); `None` reads it from `metadata.json`. The + /// caller must have made the on-disk store consistent with the override + /// (a fresh DB, or one whose data will be cleared by the reindex) — opening + /// a store at a different dimension than its vectors were written with + /// yields a dimension mismatch on the first insert. fn try_open_stores( &self, alias: &str, db_path: &Path, allow_create: bool, force_readonly: bool, + dimension_override: Option, ) -> std::result::Result { if !db_path.exists() && !allow_create { let parent = db_path @@ -2309,7 +2729,7 @@ impl ServeState { )); } - let dims = self.get_dimensions_for_path(db_path); + let dims = dimension_override.unwrap_or_else(|| self.get_dimensions_for_path(db_path)); // Read-only requested via the per-repo `repo_read_only` config flag: // open readonly directly and never attempt a write open. This makes @@ -2875,6 +3295,10 @@ impl ServeState { // deleted (the repo can be re-opened on the next query), so we // don't need to await — the cancelled task drains on its own. self.fsw_tasks.remove(alias); + // Cached C# symbol-index state must not outlive the repo (see + // clear_csharp_index_state): without this, an Error entry frozen + // from a lost double-open race renders red forever. + self.clear_csharp_index_state(alias); match self.repos.remove(alias) { Some((_, RepoState::Write { cancel_token, .. })) => { cancel_token.cancel(); @@ -2922,6 +3346,107 @@ async fn healthz_handler() -> AxumJson { AxumJson(json!({ "status": "ok" })) } +/// Query parameters for `GET /indexing`. +#[derive(serde::Deserialize)] +struct IndexingQuery { + /// Absolute filesystem path of the search target (file or directory). + path: String, +} + +/// Response body for `GET /indexing`. +/// +/// `covered=false` means the path is not inside any registered repo — the +/// caller should treat that as "no freshness signal" and behave exactly as +/// before this endpoint existed (backwards-compatible for older hooks). +#[derive(serde::Serialize)] +struct IndexingResponse { + covered: bool, + alias: Option, + indexing: bool, +} + +/// Component-boundary prefix match: does `target` lie inside `root`? +/// +/// `/x/xy` must NOT match root `/x` — comparing components (not string +/// prefixes) makes the boundary exact. On Windows the comparison is +/// case-insensitive (`repos.json` may record a different case than the +/// caller's path); on other platforms it is exact. +fn path_contains(target: &Path, root: &Path) -> bool { + let t: Vec<_> = target.components().collect(); + let r: Vec<_> = root.components().collect(); + if r.len() > t.len() { + return false; + } + let eq = |a: &std::path::Component<'_>, b: &std::path::Component<'_>| { + if a == b { + return true; + } + if cfg!(windows) { + a.as_os_str() + .to_string_lossy() + .eq_ignore_ascii_case(&b.as_os_str().to_string_lossy()) + } else { + false + } + }; + r.iter().zip(t.iter()).all(|(a, b)| eq(a, b)) +} + +/// Resolve `target` to the registered repo that contains it. +/// +/// Longest root wins, so nested registered repos (a repo inside another +/// repo's tree) resolve to the inner one. Returns `None` for paths outside +/// every registered repo (including relative paths — callers must pass +/// absolute paths). +fn containing_repo_alias( + repos: &std::collections::HashMap, + target: &Path, +) -> Option { + repos + .iter() + .filter(|(_, root)| path_contains(target, root)) + .max_by_key(|(_, root)| root.components().count()) + .map(|(alias, _)| alias.clone()) +} + +impl ServeState { + /// Freshness for an absolute filesystem path: which registered repo + /// contains it (if any), and is that repo mid-reindex right now? + /// + /// The `is_indexing` side lazily evicts stale markers, so a leaked + /// indexing task cannot report "indexing" forever. + fn freshness_for_path(&self, target: &str) -> (Option, bool) { + let config = match self.config.read() { + Ok(c) => c, + Err(_) => return (None, false), + }; + match containing_repo_alias(&config.repos, Path::new(target)) { + Some(alias) => { + let indexing = self.is_indexing(&alias); + (Some(alias), indexing) + } + None => (None, false), + } + } +} + +/// Indexing-freshness handler: GET /indexing?path= +/// +/// Lets a caller distinguish "empty result because nothing matches" from +/// "stale result because the index is mid-rebuild" — the exact distinction +/// the grep-guard hook needs after a branch switch. See [`INDEXING_PATH`]. +async fn indexing_handler( + axum::extract::State(state): axum::extract::State>, + axum::extract::Query(q): axum::extract::Query, +) -> AxumJson { + let (alias, indexing) = state.freshness_for_path(&q.path); + AxumJson(IndexingResponse { + covered: alias.is_some(), + alias, + indexing, + }) +} + /// Status handler: GET /status /// /// Returns a JSON snapshot of all repo states, active sessions, and CPU usage. @@ -2977,6 +3502,10 @@ async fn status_handler( let uptime_secs = state.started_at().elapsed().as_secs(); + // Serve-wide default model for newly created indexes (`serve --model`). + // `null` means the built-in default. + let default_model = state.default_model().map(|m| m.short_name()); + // CPU usage — reuse shared System instance so cpu_usage() can compute delta let cpu = { use sysinfo::ProcessesToUpdate; @@ -2987,6 +3516,7 @@ async fn status_handler( "version": env!("CARGO_PKG_VERSION"), "repos": repo_json, "active_sessions": active_sessions, + "default_model": default_model, "cpu_percent": "—", "uptime_secs": uptime_secs, })); @@ -2999,6 +3529,7 @@ async fn status_handler( "version": env!("CARGO_PKG_VERSION"), "repos": repo_json, "active_sessions": active_sessions, + "default_model": default_model, "cpu_percent": "—", "uptime_secs": uptime_secs, })); @@ -3031,6 +3562,7 @@ async fn status_handler( "version": env!("CARGO_PKG_VERSION"), "repos": repo_json, "active_sessions": active_sessions, + "default_model": default_model, "cpu_percent": cpu, "csharp_helper": csharp_helper, "ts_helper": ts_helper, @@ -3328,14 +3860,29 @@ async fn trigger_symbol_rebuild( state.schedule_persist_repos_config(); } Ok(Err(e)) => { - tracing::error!("❌ Symbol rebuild failed for '{}': {}", alias_owned, e); + let msg = e.to_string(); state.end_indexing(&alias_owned); state .csharp_index_error - .insert(alias_owned.clone(), e.to_string()); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + .insert(alias_owned.clone(), msg.clone()); + if ServeState::is_lmdb_format_corruption(&msg) { + tracing::warn!( + "⚠️ LMDB storage-format corruption for '{}' (data written by an older \ + storage-major) — queueing sequential wipe + full rebuild", + alias_owned + ); + // Recovery owns the outcome from here: show in-progress rather + // than Error; it flips to Ready on success or Error on failure. + state + .csharp_index_status + .insert(alias_owned.clone(), CSharpIndexStatus::Indexing); + state.enqueue_format_recovery(&alias_owned); + } else { + tracing::error!("❌ Symbol rebuild failed for '{}': {}", alias_owned, msg); + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } Err(e) => { tracing::error!( @@ -3497,7 +4044,7 @@ async fn reindex_handler( // FSW not running -- open existing or create fresh DB. // allow_create=true so a force-reindex can recover a deleted DB. let cancel = CancellationToken::new(); - match state.try_open_stores(&alias, &db_path, true, false) { + match state.try_open_stores(&alias, &db_path, true, false, None) { Ok(OpenedStores::Write(s)) => { // Register as Write to block double-open races while we reindex. state.repos.insert( @@ -3709,6 +4256,29 @@ struct AddRepoRequest { model: Option, } +/// Decide the embedding model a `POST /repos` add should index with. +/// +/// Precedence: +/// 1. an explicit `model` in the request always wins (it forces a rebuild at +/// that model's dimension, which is the documented `index add --model` +/// behavior); +/// 2. otherwise the serve-wide default (`codesearch serve --model`) applies +/// **only when no model is recorded on disk** — i.e. this call is creating a +/// brand-new index; +/// 3. an index that already records its own model keeps it, exactly as if +/// `--model` had not been passed. +fn resolve_add_repo_model( + explicit: Option, + recorded: Option, + serve_default: Option, +) -> Option { + explicit.or(if recorded.is_none() { + serve_default + } else { + None + }) +} + /// Add-repo handler: POST /repos /// /// Registers a new repo in repos.json, opens the LMDB/Tantivy stores inline @@ -3750,6 +4320,42 @@ async fn add_repo_handler( ); } + // db_path is resolved before the model decision: whether a serve-wide + // default applies depends on whether the index already records a model. + let db_path = canonical_path.join(DB_DIR_NAME); + + // Parse the optional model override BEFORE opening the store: a fresh index + // must be created at the override's dimension, not the 384-dim default. + // Previously the store was opened at the default (or the previous metadata's) + // dimension and the override was only applied to metadata afterwards, so the + // reindex embedded 768-dim vectors into a 384-dim store and indexed nothing. + let explicit_model: Option = match body.model.as_deref() { + Some(model_str) => match crate::embed::ModelType::parse(model_str) { + Some(mt) => Some(mt), + None => { + return ( + StatusCode::BAD_REQUEST, + axum::response::Json(json!({ + "error": format!("Unknown model: '{}'. Use one of: {}", model_str, crate::embed::ModelType::valid_short_names()), + "status": "error" + })), + ); + } + }, + None => None, + }; + + // `codesearch serve --model X` sets the default for indexes created here. + // It applies only when no explicit `model` was given AND this POST is + // creating a brand-new index: an existing index keeps the model recorded in + // its `metadata.json`, exactly as if the flag had not been set. An explicit + // `model` still wins and rebuilds at that model's dimension. + let model_override = resolve_add_repo_model( + explicit_model, + crate::embed::ModelType::from_index_metadata(&db_path), + state.default_model(), + ); + // Register in repos.json let alias = { let mut config = match state.config.write() { @@ -3807,8 +4413,13 @@ async fn add_repo_handler( // This eliminates the LMDB double-open race that occurred when the old // path opened its own LMDB handle, conflicting with // calls from the serve's request handlers. - let db_path = canonical_path.join(DB_DIR_NAME); - let stores = match state.try_open_stores(&alias, &db_path, true, false) { + let stores = match state.try_open_stores( + &alias, + &db_path, + true, + false, + model_override.map(|m| m.dimensions()), + ) { Ok(OpenedStores::Write(s)) => s, Ok(OpenedStores::Readonly(_)) => { unreachable!( @@ -3879,23 +4490,6 @@ async fn add_repo_handler( ); } - // Parse optional model override from request body. - let model_override: Option = match body.model.as_deref() { - Some(model_str) => match crate::embed::ModelType::parse(model_str) { - Some(mt) => Some(mt), - None => { - return ( - StatusCode::BAD_REQUEST, - axum::response::Json(json!({ - "error": format!("Unknown model: '{}'. Use one of: {}", model_str, crate::embed::ModelType::valid_short_names()), - "status": "error" - })), - ); - } - }, - None => None, - }; - // Spawn the heavy indexing work in the background. Returns 202 immediately. let alias_bg = alias.clone(); let state_bg = state.clone(); @@ -4059,7 +4653,7 @@ pub(crate) struct RepoRemovalOutcome { pub db_delete_error: Option, } -/// Remove-repo handler: DELETE /repos/:alias +/// Remove-repo handler: DELETE /repos/{alias} /// /// Stops the FSW, evicts the repo from memory, unregisters from repos.json, /// and deletes the database directory. Returns 200 on success (status is @@ -4240,8 +4834,8 @@ fn request_has_valid_api_key(headers: &axum::http::HeaderMap, configured: &str) /// /// When the env var is unset or empty, all requests pass through (backward compatible). /// -/// Management endpoints are: `POST /repos`, `DELETE /repos/:alias`, -/// `POST /repos/:alias/reindex`, `POST /reload`. +/// Management endpoints are: `POST /repos`, `DELETE /repos/{alias}`, +/// `POST /repos/{alias}/reindex`, `POST /reload`. /// All other routes (health, status, MCP) are always unauthenticated. /// /// Key comparison is constant-time (see `api_key_matches`). @@ -4693,10 +5287,16 @@ fn keep_warm_foreign_target(ping_url: &str, self_host: &str) -> Option { } } +// `run_serve` is the single startup entry point, so its parameter list is the +// serve CLI surface (bind host/port, registration, default model, TUI, +// keep-warm, shutdown). Bundling them into a struct would only move the +// plumbing; allow the wide signature instead. +#[allow(clippy::too_many_arguments)] pub async fn run_serve( host: Option, port: Option, register_paths: Vec, + default_model: Option, no_tui: bool, keep_warm_url: Option, idle_suspend_secs: Option, @@ -4780,7 +5380,7 @@ pub async fn run_serve( // env > default); nothing else consumes it, so `ServeState` does not carry // it. In particular the embedded TUI must NOT derive a poll cadence from it // — it never polls a federated peer on a timer at all. - let serve_state = Arc::new(ServeState::new(config, None)); + let serve_state = Arc::new(ServeState::new(config, None).with_default_model(default_model)); // Construct the bind address from resolved host + port. // Using `format!` with `parse::()` handles both IPv4 and IPv6. @@ -4814,6 +5414,20 @@ pub async fn run_serve( info!("📋 Registered repos: {}", repo_list); eprintln!("📋 Registered repos: {}", repo_list); + // Report the serve-wide default model for newly created indexes, if set. + // Without this, `serve --model X` is a silent setting: the TUI/status show + // per-repo models, but nothing tells an operator what a new `POST /repos` + // (or a delegated `codesearch index add`) will use. + if let Some(model) = default_model { + let line = format!( + "🧠 Default model for new indexes: {} ({} dims)", + model.short_name(), + model.dimensions() + ); + info!("{}", line); + eprintln!("{}", line); + } + // ── Start HTTP server FIRST ── // Accept connections immediately so MCP clients don't time out. // Pre-warming runs in the background below. @@ -4869,27 +5483,31 @@ pub async fn run_serve( .route(HEALTH_PATH, axum::routing::get(health_handler)) .route(HEALTHZ_PATH, axum::routing::get(healthz_handler)) .route(STATUS_PATH, axum::routing::get(status_handler)) + // Freshness probe for the grep-guard hook — same auth class as + // /status (localhost: open, network bind: bearer key). NOT in the + // always-unauthenticated set: /healthz stays the only one of those. + .route(INDEXING_PATH, axum::routing::get(indexing_handler)) // /remotes is a status-like read-only observability endpoint (lists the // configured federation peers). It is NOT in require_admin_auth's // `is_management` set, so it inherits exactly the same auth policy as - // /status, /repos/:alias/info and /repos/:alias/doctor: reachable + // /status, /repos/{alias}/info and /repos/{alias}/doctor: reachable // without the admin key on localhost, protected by // require_auth_for_network on network binds. See REMOTES_PATH doc. .route(REMOTES_PATH, axum::routing::get(remotes_handler)) .route("/repos", axum::routing::post(add_repo_handler)) - .route("/repos/:alias", axum::routing::delete(remove_repo_handler)) + .route("/repos/{alias}", axum::routing::delete(remove_repo_handler)) .route("/reload", axum::routing::post(reload_handler)) .route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post(reindex_handler), ) - .route("/repos/:alias/info", axum::routing::get(info_handler)) + .route("/repos/{alias}/info", axum::routing::get(info_handler)) // /doctor is a POST but is intentionally read-only (diagnostics only, no // --fix path), so like /info and /status it is NOT in require_admin_auth's // management set — reachable without the admin key on localhost, and still // protected by require_auth_for_network on network binds. If doctor ever // gains a mutating mode, add it to `is_management` in require_admin_auth. - .route("/repos/:alias/doctor", axum::routing::post(doctor_handler)) + .route("/repos/{alias}/doctor", axum::routing::post(doctor_handler)) // REST endpoints — federation-friendly HTTP+JSON mirror of the read-only // MCP tools (search/find/explore/get_chunk). Lets a remote codesearch // serve be queried WITHOUT an MCP session. Same auth layers as /mcp & @@ -4911,6 +5529,10 @@ pub async fn run_serve( CHUNK_PATH, axum::routing::get(crate::mcp::rest_get_chunk_handler), ) + .route( + FIND_IMPACT_PATH, + axum::routing::post(crate::mcp::rest_find_impact_handler), + ) .nest_service(MCP_ENDPOINT_PATH, mcp_service) .layer(axum::middleware::from_fn(require_admin_auth)) .layer(axum::middleware::from_fn(log_mcp_requests)) diff --git a/src/serve/tests.rs b/src/serve/tests.rs index fbc31f18..53081376 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -1,4 +1,5 @@ use super::*; +use serial_test::serial; use std::io::Write; #[test] @@ -29,6 +30,62 @@ fn rest_service_drop_does_not_touch_active_sessions() { ); } +#[tokio::test] +#[serial] +async fn get_chunk_routes_mounted_remote_projects_through_federation() { + // todo #153: `get_chunk(project="/", chunk_id=…)` must route + // through the federated fetch exactly like search's project-level + // federation, instead of dying in local routing with "Unknown alias". + // The peer URL here is unreachable, so the correctly routed answer is the + // federation failure message — "Unknown alias" means the routing did not + // happen. + let mut config = ReposConfig::default(); + config.remotes.insert( + "cloud".to_string(), + crate::db_discovery::repos::RemotePeer { + url: "http://127.0.0.1:1".to_string(), + api_key: "test-key".to_string(), + group: None, + timeout_secs: None, + }, + ); + config.remote_mounts.push("cloud/bynder".to_string()); + // Hermetic config: persist to a temp file and pass the override, so + // `reload_if_changed` reads THIS config — not the developer's real + // ~/.codesearch/repos.json (which would leak real peers into the test). + let tmp = tempfile::tempdir().unwrap(); + let config_file = tmp.path().join("repos.json"); + config.save_to(&config_file).unwrap(); + let state = std::sync::Arc::new(ServeState::new(config, Some(config_file))); + let service = crate::mcp::CodesearchService::new_for_serve(state).unwrap(); + + let _env = + crate::testing::EnvRestore::set(&[(crate::constants::REMOTE_PEER_RETRY_BACKOFF_ENV, "1")]); + let req = crate::mcp::types::GetChunkRequest { + chunk_id: 2058, + chunk_ref: None, + context_lines: None, + project: Some("cloud/bynder".to_string()), + group: None, + }; + let res = service + .get_chunk(rmcp::handler::server::wrapper::Parameters(req)) + .await + .expect("handler must not error"); + let text = match res.content.first() { + Some(rmcp::model::ContentBlock::Text(t)) => t.text.clone(), + other => panic!("expected text content, got {other:?}"), + }; + assert!( + text.contains("Could not fetch chunk from remote peer 'cloud'"), + "get_chunk must route mounted remote projects to the peer, got: {text}" + ); + assert!( + !text.contains("Unknown alias"), + "a mounted remote project is not a local alias — routing failed: {text}" + ); +} + #[test] fn tracked_session_drop_balances_active_sessions() { // A genuine MCP session increments on connect and the serve factory @@ -302,6 +359,80 @@ fn is_db_locked_error_classifies_lock_and_non_lock_errors() { ))); } +/// Open a real registered LMDB env at `path` — the same holder shape +/// `remove_repo`'s lock-class retry waits on (a live `TrackedEnv` keeps the +/// mmap file handles on Windows and its registry slot everywhere). +fn open_test_lmdb_env( + path: &std::path::Path, + description: &str, +) -> crate::lmdb_registry::TrackedEnv { + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(1024 * 1024).max_dbs(1); + unsafe { opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS) }; + unsafe { crate::lmdb_registry::TrackedEnv::open(&opts, path, description).unwrap() } +} + +/// `await_lmdb_release` returns empty once the last in-process holder drops, +/// and does NOT return early while the holder is alive. A spawned dropper +/// releases the env after 200 ms; the helper (deadline 5 s) must observe the +/// drain. The elapsed lower bound only rules out an instant-return bug — +/// it cannot flake, since the env provably lived that long. +#[tokio::test] +async fn await_lmdb_release_drains_after_holder_drops() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + + let env = open_test_lmdb_env(&db_path, "transient-search-holder"); + let held_from = std::time::Instant::now(); + // Drop the env from a spawned task after a short delay — mimics an + // in-flight search finishing and dropping its Arc. + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + drop(env); + }); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let remaining = ServeState::await_lmdb_release(&db_path, deadline).await; + + assert!( + remaining.is_empty(), + "helper must report a full drain, still sees: {remaining:?}" + ); + assert!( + held_from.elapsed() >= std::time::Duration::from_millis(200), + "helper returned before the holder actually dropped — early-return bug" + ); +} + +/// The budget-expiry arm: a holder that never releases must not hang the +/// helper past its deadline — it returns the surviving holder descriptions so +/// `remove_repo` can log exactly who outlived the budget. +#[tokio::test] +async fn await_lmdb_release_returns_holders_at_deadline() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + + let _env = open_test_lmdb_env(&db_path, "stuck-embed-pass"); + + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(150); + // Outer bound so a regression to an unbounded loop fails the test fast + // instead of hanging the suite. + let remaining = tokio::time::timeout( + std::time::Duration::from_secs(2), + ServeState::await_lmdb_release(&db_path, deadline), + ) + .await + .expect("helper must return at its deadline, not hang"); + + assert_eq!( + remaining, + vec!["stuck-embed-pass".to_string()], + "deadline expiry must carry the surviving holder's description" + ); +} + #[test] fn is_alias_live_reflects_config_and_cancellation() { // FINDINGS #4: the resurrection guard. A detached indexing task must @@ -464,6 +595,80 @@ async fn missing_db_not_cached_as_conflicted() { assert!(res.is_ok(), "expected ok after recreating DB, got: Err"); } +/// Pin for the cold-open single-flight (todo #131): a second opener must PARK +/// on the per-alias open lock instead of racing into `try_open_stores` and +/// tripping the LMDB double-open guard. Deterministic: every step before the +/// lock acquire is synchronous, so one yield lets the spawned task reach the +/// lock await; on revert (no single-flight) the whole call completes on that +/// same poll and the `!is_finished()` assertion fails. +#[tokio::test] +async fn cold_open_parks_second_opener_behind_alias_lock() { + let (_tmp, _repo_path, state) = state_with_repo("testalias"); + let state = std::sync::Arc::new(state); + + // Hold the alias open lock like an in-flight cold open would. + let alias_lock = state.open_lock("testalias"); + let guard = alias_lock.lock().await; + + let s2 = std::sync::Arc::clone(&state); + let task = tokio::spawn(async move { s2.get_or_open_stores("testalias", true).await }); + + tokio::task::yield_now().await; + assert!( + !task.is_finished(), + "second opener must park on the alias open lock, not run (and fail) concurrently" + ); + + drop(guard); + let res = task.await.unwrap(); + // No DB seeded → the parked opener resumes and fails with the ordinary + // missing-DB error; what matters is that it ran to completion AFTER the + // lock was released, never concurrently. + assert!( + res.is_err(), + "expected the ordinary missing-DB error after the lock was released" + ); +} + +/// Invariant: N concurrent cold opens of the same repo must all succeed and +/// share ONE stores Arc (single open, everyone else hits the cache re-check). +/// Pre-single-flight this race could produce the LMDB double-open error on +/// the losers; with it, exactly one opener reaches `try_open_stores`. +#[tokio::test] +async fn concurrent_cold_opens_share_one_stores_arc() { + let (_tmp, repo_path, state) = state_with_repo("testalias"); + // Seed an openable DB (same recipe as missing_db_not_cached_as_conflicted). + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir(&db_path).unwrap(); + let mut f = std::fs::File::create(db_path.join("metadata.json")).unwrap(); + write!(f, "{{\"dimensions\":384}}").unwrap(); + drop(f); + + let state = std::sync::Arc::new(state); + let mut handles = Vec::new(); + for _ in 0..8 { + let s = std::sync::Arc::clone(&state); + handles.push(tokio::spawn(async move { + s.get_or_open_stores("testalias", true).await + })); + } + + let mut first: Option> = None; + for h in handles { + let stores = h + .await + .unwrap() + .expect("concurrent cold open must not fail (double-open guard)"); + match &first { + None => first = Some(stores), + Some(fst) => assert!( + std::sync::Arc::ptr_eq(fst, &stores), + "concurrent openers must share one stores Arc" + ), + } + } +} + #[tokio::test] async fn not_found_error_mentions_fix_commands() { let tmp = tempfile::tempdir().unwrap(); @@ -621,7 +826,7 @@ async fn try_open_stores_creates_db_for_brand_new_repo() { let state = state_with_config(ReposConfig::default()); - match state.try_open_stores("brandnew", &db_path, true, false) { + match state.try_open_stores("brandnew", &db_path, true, false, None) { Ok(OpenedStores::Write(_)) => {} Ok(OpenedStores::Readonly(_)) => { panic!("brand-new repo opened Readonly; expected Write") @@ -648,8 +853,15 @@ async fn try_open_stores_creates_db_for_brand_new_repo() { /// handler's synchronous pre-spawn state — no embedding model required, no /// race. `persist_config` honors the temp config override, so the real /// `~/.codesearch/repos.json` is never touched. +/// +/// `#[serial]` + env reset: the handler reads `CODESEARCH_ALLOWED_ROOTS` via +/// `validate_path_within_allowed_roots`, so this test must not run while the +/// `allowed_roots_tests` below are mutating it (and must not inherit a stale +/// value from ambient state). +#[serial] #[tokio::test] async fn add_repo_handler_registers_brand_new_repo_without_rollback() { + let _env = crate::testing::EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let tmp = tempfile::tempdir().unwrap(); let repo_path = tmp.path().join("brandnew"); std::fs::create_dir(&repo_path).unwrap(); @@ -695,6 +907,250 @@ async fn add_repo_handler_registers_brand_new_repo_without_rollback() { ); } +/// `POST /repos` with no explicit `model` must create a brand-new index at the +/// serve-wide default's dimension (`codesearch serve --model X`), not the +/// built-in 384-dim default. This is the write-side counterpart of the per-repo +/// query-model contract. +#[tokio::test] +async fn add_repo_handler_uses_serve_default_model_for_new_index() { + let tmp = tempfile::tempdir().unwrap(); + let repo_path = tmp.path().join("defaulted"); + std::fs::create_dir(&repo_path).unwrap(); + + let state = Arc::new( + state_with_config(ReposConfig::default()) + .with_default_model(Some(crate::embed::ModelType::EmbeddingGemma300MQ4)), + ); + + let (status, body) = add_repo_handler( + axum::extract::State(state.clone()), + axum::extract::Json(AddRepoRequest { + path: repo_path.clone(), + alias: Some("defaulted".to_string()), + model: None, + }), + ) + .await; + assert_eq!( + status, + axum::http::StatusCode::ACCEPTED, + "add must be accepted, got {}: {}", + status, + body.0 + ); + + let stores = state + .get_opened_stores("defaulted") + .expect("store must be open immediately after add"); + let dims = stores + .vector_store + .try_read() + .unwrap() + .stats() + .unwrap() + .dimensions; + assert_eq!( + dims, + crate::embed::ModelType::EmbeddingGemma300MQ4.dimensions(), + "a new index must be created at the serve default model's dimension" + ); +} + +/// The serve-wide default must NOT override an index that already records its +/// own model: re-adding a repo whose `.codesearch.db` is still on disk keeps the +/// recorded model and dimension. +#[tokio::test] +async fn add_repo_handler_keeps_recorded_model_over_serve_default() { + let tmp = tempfile::tempdir().unwrap(); + let repo_path = tmp.path().join("existing"); + std::fs::create_dir(&repo_path).unwrap(); + let db_path = repo_path.join(DB_DIR_NAME); + + // A pre-existing index recording the 384-dim default model. + std::fs::create_dir_all(&db_path).unwrap(); + let mut meta = serde_json::Map::new(); + crate::embed::ModelType::AllMiniLML6V2Q.write_metadata_fields(&mut meta); + std::fs::write( + db_path.join("metadata.json"), + serde_json::to_string(&meta).unwrap(), + ) + .unwrap(); + + let state = Arc::new( + state_with_config(ReposConfig::default()) + .with_default_model(Some(crate::embed::ModelType::EmbeddingGemma300MQ4)), + ); + + let (status, body) = add_repo_handler( + axum::extract::State(state.clone()), + axum::extract::Json(AddRepoRequest { + path: repo_path.clone(), + alias: Some("existing".to_string()), + model: None, + }), + ) + .await; + assert_eq!( + status, + axum::http::StatusCode::ACCEPTED, + "add must be accepted, got {}: {}", + status, + body.0 + ); + + let stores = state + .get_opened_stores("existing") + .expect("store must be open immediately after add"); + let dims = stores + .vector_store + .try_read() + .unwrap() + .stats() + .unwrap() + .dimensions; + assert_eq!( + dims, + crate::embed::ModelType::AllMiniLML6V2Q.dimensions(), + "an existing index must keep its recorded model, not adopt the serve default" + ); +} + +/// Precedence contract for the model a `POST /repos` add indexes with. +#[test] +fn resolve_add_repo_model_precedence() { + use crate::embed::ModelType; + let gemma = ModelType::EmbeddingGemma300MQ4; + let mini = ModelType::AllMiniLML6V2Q; + + // Explicit model always wins, even over a recorded model and a default. + assert_eq!( + resolve_add_repo_model(Some(gemma), Some(mini), Some(mini)), + Some(gemma) + ); + // No explicit model, no recorded model → serve default applies (new index). + assert_eq!(resolve_add_repo_model(None, None, Some(gemma)), Some(gemma)); + // No explicit model, recorded model present → serve default is ignored. + assert_eq!(resolve_add_repo_model(None, Some(mini), Some(gemma)), None); + // No explicit model, no recorded model, no default → no override. + assert_eq!(resolve_add_repo_model(None, None, None), None); + // Explicit model still wins when nothing else is set. + assert_eq!(resolve_add_repo_model(Some(mini), None, None), Some(mini)); +} + +/// The serve-wide default is the scope-free fallback model in serve mode (the +/// unpinned status summary, a call with no routed alias). It is deliberately +/// NOT the query fallback for a repo that records no model — see +/// `unrecorded_index_is_queried_with_builtin_default_not_serve_default`. +#[test] +fn serve_default_model_is_service_fallback() { + use crate::embed::ModelType; + let state = std::sync::Arc::new( + ServeState::new(ReposConfig::default(), None) + .with_default_model(Some(ModelType::EmbeddingGemma300MQ4)), + ); + assert_eq!(state.default_model(), Some(ModelType::EmbeddingGemma300MQ4)); + + let svc = crate::mcp::CodesearchService::new_for_serve(state).unwrap(); + assert_eq!( + svc.query_model(None), + ModelType::EmbeddingGemma300MQ4, + "serve must fall back to its default model, not the built-in default" + ); +} + +/// Without `--model`, `ServeState` reports no default and the service falls back +/// to the built-in default. +#[test] +fn no_serve_default_keeps_builtin_fallback() { + use crate::embed::ModelType; + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + assert_eq!(state.default_model(), None); + + let svc = crate::mcp::CodesearchService::new_for_serve(state).unwrap(); + assert_eq!(svc.query_model(None), ModelType::default()); +} + +/// A repo whose `metadata.json` records no model is queried with the BUILT-IN +/// default, never the serve-wide `--model` default. +/// +/// Regression guard for `serve --model X` silently overriding a legacy index: +/// with a 768-dim serve default, a 384-dim legacy index failed every search with +/// "Query embedding dimension mismatch: expected 384, got 768", and a +/// same-dimension default would have compared incomparable vector spaces without +/// erroring. The assumption must also reach the caller as a warning naming the +/// repo, the assumed model and the re-index command. +#[test] +fn unrecorded_index_is_queried_with_builtin_default_not_serve_default() { + use crate::embed::ModelType; + + let tmp = tempfile::tempdir().unwrap(); + let config_file = tmp.path().join("repos.json"); + let repo_path = tmp.path().join("legacy"); + std::fs::create_dir(&repo_path).unwrap(); + let mut config = ReposConfig::default(); + config + .register_with_alias(repo_path.clone(), Some("legacy".to_string())) + .unwrap(); + config.save_to(&config_file).unwrap(); + + let state = std::sync::Arc::new( + ServeState::new(config, Some(config_file)) + .with_default_model(Some(ModelType::EmbeddingGemma300MQ4)), + ); + let svc = crate::mcp::CodesearchService::new_for_serve(state).unwrap(); + + // No metadata.json yet: an unrecorded model. Serve default is gemma. + let resolution = svc.resolve_query_model(Some("legacy")); + assert_eq!( + resolution.model, + ModelType::default(), + "a repo that records no model must be queried with the built-in default, \ + not the '{}' serve default", + ModelType::EmbeddingGemma300MQ4.short_name() + ); + let warning = resolution + .assumed_warning + .expect("the assumed model must be surfaced to the caller"); + assert!( + warning.contains("legacy"), + "warning must name the repo: {warning}" + ); + assert!( + warning.contains(ModelType::default().short_name()), + "warning must name the assumed model: {warning}" + ); + assert!( + warning.contains("--force"), + "warning must give the re-index command: {warning}" + ); + + // A recorded model is used as-is and must not warn. + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write( + db_path.join("metadata.json"), + r#"{"model_short_name":"embeddinggemma-q4","dimensions":768}"#, + ) + .unwrap(); + let resolution = svc.resolve_query_model(Some("legacy")); + assert_eq!(resolution.model, ModelType::EmbeddingGemma300MQ4); + assert!( + resolution.assumed_warning.is_none(), + "a recorded model must not warn" + ); +} + +/// The unrecorded-model log warning fires once per alias, so a busy hub does not +/// repeat the same line on every query. The caller-facing warning is separate +/// and is not deduped. +#[test] +fn legacy_model_warning_is_logged_once_per_alias() { + let state = ServeState::new(ReposConfig::default(), None); + assert!(state.mark_legacy_model_warned("a")); + assert!(!state.mark_legacy_model_warned("a")); + assert!(state.mark_legacy_model_warned("b")); +} + /// `persist_config` must write to the override path (and therefore be /// observable by `reload_if_changed`/`config_snapshot`) rather than the real /// `~/.codesearch/repos.json`. Guards the wiring that makes the register @@ -834,10 +1290,15 @@ fn config_reload_no_spurious_reload() { assert_eq!(after_second, after_first); } -/// Verify that the /repos/:alias/reindex route is registered and reachable. +/// Verify that the /repos/{alias}/reindex route is registered and reachable. /// This test starts a real axum server on a random port and sends a POST request. +/// +/// `#[serial]` + env reset — same allowed-roots race guard as the add_repo +/// handler test above. +#[serial] #[tokio::test] async fn reindex_route_is_registered() { + let _env = crate::testing::EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let tmp = tempfile::tempdir().unwrap(); let repo_path = tmp.path().join("myrepo"); std::fs::create_dir(&repo_path).unwrap(); @@ -858,7 +1319,7 @@ async fn reindex_route_is_registered() { axum::routing::get(health_handler), ) .route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post(reindex_handler), ) .with_state(state); @@ -925,8 +1386,10 @@ async fn reindex_route_is_registered() { /// corpus index it only holds read-only. The handler returns 409 CONFLICT with /// `status: "read_only"` (see the read-only guard in `reindex_handler`, /// src/serve/mod.rs). +#[serial] #[tokio::test] async fn reindex_refused_for_read_only_repo_even_with_force() { + let _env = crate::testing::EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let (_tmp, _repo_path, state) = state_with_repo("readonlyrepo"); // Mark the repo read-only in the live config (how a snapshot-restore sets it). state @@ -943,7 +1406,7 @@ async fn reindex_refused_for_read_only_repo_even_with_force() { axum::routing::get(health_handler), ) .route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post(reindex_handler), ) .with_state(state); @@ -1033,7 +1496,7 @@ async fn healthz_is_unauthenticated_on_network_bind() { ); } -/// Verify that the /repos/:alias/info and /repos/:alias/doctor routes are +/// Verify that the /repos/{alias}/info and /repos/{alias}/doctor routes are /// registered and reachable. Starts a real axum server on a random port and /// asserts that an unknown alias yields our handler's 404 (not axum's 404). #[tokio::test] @@ -1057,8 +1520,8 @@ async fn info_doctor_routes_registered() { crate::constants::HEALTH_PATH, axum::routing::get(health_handler), ) - .route("/repos/:alias/info", axum::routing::get(info_handler)) - .route("/repos/:alias/doctor", axum::routing::post(doctor_handler)) + .route("/repos/{alias}/info", axum::routing::get(info_handler)) + .route("/repos/{alias}/doctor", axum::routing::post(doctor_handler)) .with_state(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -1146,7 +1609,7 @@ async fn info_doctor_routes_registered() { } /// Verify that the federation REST endpoints (/search, /find, /explore, -/// /chunk/:id) are registered and reachable. Each must dispatch to OUR +/// /chunk/{id}) are registered and reachable. Each must dispatch to OUR /// handler (returning a JSON body) rather than axum's built-in empty 404. /// Starts a real axum server on a random port. #[tokio::test] @@ -1186,6 +1649,10 @@ async fn rest_routes_are_registered() { crate::constants::CHUNK_PATH, axum::routing::get(crate::mcp::rest_get_chunk_handler), ) + .route( + crate::constants::FIND_IMPACT_PATH, + axum::routing::post(crate::mcp::rest_find_impact_handler), + ) .with_state(state); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -1257,6 +1724,24 @@ async fn rest_routes_are_registered() { .await .expect("POST /find should return JSON from our handler"); + // POST /find-impact — dispatches to rest_find_impact_handler. + let resp = client + .post(format!("http://{}/find-impact", addr)) + .json(&serde_json::json!({"symbol_name": "foo", "project": "testalias"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == reqwest::StatusCode::OK + || resp.status() == reqwest::StatusCode::INTERNAL_SERVER_ERROR, + "POST /find-impact -> unexpected status {} (route not registered?)", + resp.status() + ); + let _: serde_json::Value = resp + .json() + .await + .expect("POST /find-impact should return JSON from our handler"); + // POST /explore — dispatches to rest_explore_handler. let resp = client .post(format!("http://{}/explore", addr)) @@ -1327,7 +1812,7 @@ async fn concurrent_reindex_returns_conflict() { let app = axum::Router::new() .route( - "/repos/:alias/reindex", + "/repos/{alias}/reindex", axum::routing::post(reindex_handler), ) .with_state(state); @@ -1377,20 +1862,17 @@ async fn concurrent_reindex_returns_conflict() { /// Unit tests for `validate_path_within_allowed_roots`. /// -/// These tests temporarily set/remove the `CODESEARCH_ALLOWED_ROOTS` env var. -/// A static Mutex serializes env mutation to prevent races under parallel test execution. +/// These tests mutate the `CODESEARCH_ALLOWED_ROOTS` env var. Per the +/// AGENTS.md rule they are `#[serial]` and restore the var via `EnvRestore`: +/// a private Mutex cannot protect against non-serial tests elsewhere in the +/// process that READ the var through the real handlers (the add_repo +/// handler test below), which is exactly the 403 flake this closed. #[cfg(test)] mod allowed_roots_tests { use super::*; + use crate::testing::EnvRestore; + use serial_test::serial; use std::path::PathBuf; - use std::sync::Mutex; - - /// Global lock to serialize env var mutations across parallel test threads. - static ENV_LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); - - fn lock() -> std::sync::MutexGuard<'static, ()> { - ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap() - } /// Helper: create a unique temp dir per test, return its canonical path. fn temp_root(suffix: &str) -> PathBuf { @@ -1399,57 +1881,49 @@ mod allowed_roots_tests { safe_canonicalize(&dir).unwrap() } - fn clear_env() { - std::env::remove_var(ALLOWED_ROOTS_ENV); - } - - fn set_env(val: &str) { - std::env::set_var(ALLOWED_ROOTS_ENV, val); - } - + #[serial] #[test] fn env_unset_allows_all() { - let _guard = lock(); - clear_env(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let path = PathBuf::from("/some/random/path"); assert!(validate_path_within_allowed_roots(&path).is_ok()); } + #[serial] #[test] fn env_empty_allows_all() { - let _guard = lock(); - set_env(""); + let _env = EnvRestore::set(&[(ALLOWED_ROOTS_ENV, "")]); let path = PathBuf::from("/some/random/path"); assert!(validate_path_within_allowed_roots(&path).is_ok()); - clear_env(); } + #[serial] #[test] fn path_within_root_is_allowed() { - let _guard = lock(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let root = temp_root("within"); - set_env(&root.display().to_string()); + std::env::set_var(ALLOWED_ROOTS_ENV, root.display().to_string()); let child = root.join("my-project"); let _ = std::fs::create_dir_all(&child); let canonical_child = safe_canonicalize(&child).unwrap(); assert!(validate_path_within_allowed_roots(&canonical_child).is_ok()); - clear_env(); } + #[serial] #[test] fn exact_root_match_is_allowed() { - let _guard = lock(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let root = temp_root("exact"); - set_env(&root.display().to_string()); + std::env::set_var(ALLOWED_ROOTS_ENV, root.display().to_string()); assert!(validate_path_within_allowed_roots(&root).is_ok()); - clear_env(); } + #[serial] #[test] fn path_outside_root_is_rejected() { - let _guard = lock(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let root = temp_root("outside"); - set_env(&root.display().to_string()); + std::env::set_var(ALLOWED_ROOTS_ENV, root.display().to_string()); // Construct a path guaranteed outside the temp root let outside = if cfg!(windows) { PathBuf::from("C:\\Windows\\System32") @@ -1465,40 +1939,45 @@ mod allowed_roots_tests { let result = validate_path_within_allowed_roots(&outside); assert!(result.is_err(), "Expected rejection for path outside root"); assert!(result.unwrap_err().contains("outside allowed roots")); - clear_env(); } + #[serial] #[test] fn all_nonexistent_roots_rejects() { - let _guard = lock(); - set_env("/nonexistent/path/abc;/also/nonexistent/xyz"); + let _env = EnvRestore::set(&[( + ALLOWED_ROOTS_ENV, + "/nonexistent/path/abc;/also/nonexistent/xyz", + )]); let some_path = std::env::temp_dir(); let canonical = safe_canonicalize(&some_path).unwrap(); let result = validate_path_within_allowed_roots(&canonical); assert!(result.is_err()); assert!(result.unwrap_err().contains("No valid roots found")); - clear_env(); } + #[serial] #[test] fn semicolons_with_empty_segments_works() { - let _guard = lock(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let root = temp_root("semicolons"); - set_env(&format!(";{};;", root.display())); + std::env::set_var(ALLOWED_ROOTS_ENV, format!(";{};;", root.display())); let child = root.join("project"); let _ = std::fs::create_dir_all(&child); let canonical_child = safe_canonicalize(&child).unwrap(); assert!(validate_path_within_allowed_roots(&canonical_child).is_ok()); - clear_env(); } + #[serial] #[test] fn multiple_roots_any_match() { - let _guard = lock(); + let _env = EnvRestore::remove(&[ALLOWED_ROOTS_ENV]); let root1 = temp_root("multi1"); let root2 = temp_root("multi2"); - set_env(&format!("{};{}", root1.display(), root2.display())); + std::env::set_var( + ALLOWED_ROOTS_ENV, + format!("{};{}", root1.display(), root2.display()), + ); // Path under root1 let child1 = root1.join("project"); @@ -1511,8 +1990,6 @@ mod allowed_roots_tests { let _ = std::fs::create_dir_all(&child2); let canonical2 = safe_canonicalize(&child2).unwrap(); assert!(validate_path_within_allowed_roots(&canonical2).is_ok()); - - clear_env(); } } @@ -1661,7 +2138,7 @@ mod allowed_hosts_tests { /// Tests for `extract_host_from_url` — used solely by the keep-warm /// misconfiguration sanity check (a keep-warm target host that doesn't look /// like "self" gets a loud warning; see the diagnosis this shipped with in -/// docs/diagnose-federated-keep-warm.md). +/// `.docs/DIAGNOSE_FEDERATED_KEEP_WARM.md`). mod keep_warm_host_extraction_tests { use super::*; @@ -1785,3 +2262,679 @@ mod keep_warm_foreign_target_tests { ); } } + +// =========================================================================== +// GET /indexing — freshness probe (grep-guard wait-and-retry, todo #55) +// =========================================================================== + +/// Helper: a repos map from (alias, root) pairs. +fn repos_map(entries: &[(&str, &str)]) -> std::collections::HashMap { + entries + .iter() + .map(|(a, p)| (a.to_string(), std::path::PathBuf::from(p))) + .collect() +} + +#[test] +fn containing_repo_alias_matches_subdir_but_not_sibling_prefix() { + let repos = repos_map(&[("alpha", "/base/alpha"), ("beta", "/base/beta")]); + + // Exact root. + assert_eq!( + containing_repo_alias(&repos, Path::new("/base/alpha")), + Some("alpha".to_string()) + ); + // File inside the repo. + assert_eq!( + containing_repo_alias(&repos, Path::new("/base/alpha/src/main.rs")), + Some("alpha".to_string()) + ); + // Component boundary: /base/alpha-x is NOT inside /base/alpha. + assert_eq!( + containing_repo_alias(&repos, Path::new("/base/alpha-x/file.rs")), + None, + "string-prefix sibling must not match" + ); + // Entirely outside. + assert_eq!(containing_repo_alias(&repos, Path::new("/elsewhere")), None); +} + +#[test] +fn containing_repo_alias_prefers_nested_repo() { + // Two registered repos, one nested inside the other's tree: the inner + // (longer root) must win so the freshness answer is about the repo the + // path actually belongs to. + let repos = repos_map(&[("outer", "/base"), ("inner", "/base/inner")]); + assert_eq!( + containing_repo_alias(&repos, Path::new("/base/inner/src/a.rs")), + Some("inner".to_string()) + ); + assert_eq!( + containing_repo_alias(&repos, Path::new("/base/other/src/b.rs")), + Some("outer".to_string()) + ); +} + +#[test] +fn containing_repo_alias_case_insensitive_only_on_windows() { + let repos = repos_map(&[("alpha", "/Base/Alpha")]); + let hit = containing_repo_alias(&repos, Path::new("/base/alpha/x.rs")); + if cfg!(windows) { + assert_eq!(hit, Some("alpha".to_string())); + } else { + assert_eq!(hit, None, "unix path matching stays case-sensitive"); + } +} + +#[test] +fn freshness_for_path_reports_indexing_state() { + let tmp = tempfile::tempdir().unwrap(); + let repo_path = tmp.path().join("repo"); + + let mut config = ReposConfig::default(); + config + .register_with_alias(repo_path.clone(), Some("fresh".to_string())) + .unwrap(); + + let state = Arc::new(ServeState::new(config, None)); + let target = repo_path.join("src").join("lib.rs"); + + // Idle: covered, not indexing. + let (alias, indexing) = state.freshness_for_path(&target.to_string_lossy()); + assert_eq!(alias.as_deref(), Some("fresh")); + assert!(!indexing); + + // Mid-reindex (the exact state a branch-switch full refresh puts the + // repo in — make_indexing_status_callback inserts around it). + state.begin_indexing("fresh"); + let (_, indexing) = state.freshness_for_path(&target.to_string_lossy()); + assert!(indexing, "begin_indexing must surface as indexing=true"); + + state.end_indexing("fresh"); + let (_, indexing) = state.freshness_for_path(&target.to_string_lossy()); + assert!(!indexing); + + // Unknown path: not covered, no crash. + let (alias, indexing) = state.freshness_for_path("/definitely/not/registered"); + assert_eq!(alias, None); + assert!(!indexing); +} + +#[tokio::test] +async fn indexing_route_answers_json() { + // Route + handler wiring: a GET with a covered path returns our JSON + // (never axum's empty 404), with the covered/indexing fields present. + let tmp = tempfile::tempdir().unwrap(); + let raw_repo = tmp.path().join("hooked"); + std::fs::create_dir(&raw_repo).unwrap(); + // CANONICALIZE (same trap as remove_order_tests' make_proj): on the + // Windows CI runner the temp root sits under an 8.3 short name + // (RUNNER~1) that only canonicalize resolves, and register canonicalizes + // before storing — querying with the raw path made covered=false there + // (green locally, red on CI). + let repo_path = crate::cache::safe_canonicalize(&raw_repo).unwrap(); + + let mut config = ReposConfig::default(); + config + .register_with_alias(repo_path.clone(), Some("hooked".to_string())) + .unwrap(); + + let state = Arc::new(ServeState::new(config, None)); + + let app = axum::Router::new() + .route( + crate::constants::INDEXING_PATH, + axum::routing::get(indexing_handler), + ) + .with_state(state); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let client = reqwest::Client::new(); + // Temp paths are safe ASCII; normalise backslashes so the query string is + // legal as-is (component matching treats / and \ identically on Windows). + let covered = repo_path.to_string_lossy().replace('\\', "/"); + + // Covered path. + let resp = client + .get(format!( + "http://{addr}/indexing?path={covered}", + addr = addr + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), reqwest::StatusCode::OK); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["covered"], serde_json::json!(true)); + assert_eq!(body["alias"], serde_json::json!("hooked")); + assert_eq!(body["indexing"], serde_json::json!(false)); + + // Uncovered path. + let resp = client + .get(format!( + "http://{addr}/indexing?path=/nowhere/at/all", + addr = addr + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), reqwest::StatusCode::OK); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["covered"], serde_json::json!(false)); + assert_eq!(body["indexing"], serde_json::json!(false)); +} + +// =========================================================================== +// `codesearch index rm` → running serve: end-to-end delegation (todo #48 L2) +// =========================================================================== + +/// Shared envelope for the Layer-2 e2e tests below: pin the delegation env +/// vars to a fresh temp repos.json, seed it via `seed`, spawn a REAL serve +/// (the two routes the CLI `index rm` delegation touches: `GET /health` and +/// the real `remove_repo_handler` at `DELETE /repos/{alias}`) sharing that +/// config, and wait (bounded, panicking on timeout) for it to accept. +/// +/// Every step here is trap-sensitive, which is why it is a helper and not +/// copy-paste: env vars must be pinned BEFORE `cfg.save()` or the seed lands +/// in the developer's REAL registry; the port must come from +/// `listener.local_addr().port()` (never the SocketAddr — its Display form +/// makes the env var unparseable, the port silently falls back to the +/// DEFAULT 39725, and the delegation fires a live DELETE at a developer's +/// running serve); `SERVE_HOST_ENV` must be pinned to loopback or a stray +/// machine-level value sends the probe elsewhere entirely. +/// +/// Returns `(state, port, env_guard)`. The caller MUST keep the +/// [`crate::testing::EnvRestore`] guard bound for the whole test — dropping +/// it (e.g. letting a helper-internal guard die) unpins the vars before the +/// delegation runs. This is why the guard is returned rather than held here. +/// The helper lives in this module rather than `src/testing.rs` because the +/// handlers it routes are private to `serve`. +async fn spawn_rm_delegation_test_serve( + tmp: &std::path::Path, + seed: F, +) -> (Arc, u16, crate::testing::EnvRestore) +where + F: FnOnce(&mut ReposConfig), +{ + let cfg_path = tmp.join("repos.json"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + // Env vars FIRST, then seed repos.json (ordering trap: see doc above). + let env = crate::testing::EnvRestore::set(&[ + ( + crate::constants::REPOS_CONFIG_ENV, + &cfg_path.to_string_lossy(), + ), + (crate::constants::SERVE_PORT_ENV, port.to_string().as_str()), + (crate::constants::SERVE_HOST_ENV, "127.0.0.1"), + ]); + let mut cfg = ReposConfig::default(); + seed(&mut cfg); + cfg.save().expect("seed repos.json save must succeed"); + assert!( + cfg_path.exists(), + "seed repos.json must land in the temp path, not the global default" + ); + + let state = Arc::new(ServeState::new(cfg, Some(cfg_path.clone()))); + let app = axum::Router::new() + .route( + crate::constants::HEALTH_PATH, + axum::routing::get(health_handler), + ) + .route("/repos/{alias}", axum::routing::delete(remove_repo_handler)) + .with_state(state.clone()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + // Bounded readiness wait that FAILS LOUDLY on timeout (a silent exit + // here surfaces downstream as an unrelated delegation failure). + let mut ready = false; + for _ in 0..200 { + if tokio::net::TcpStream::connect(format!("127.0.0.1:{port}")) + .await + .is_ok() + { + ready = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + assert!(ready, "test serve never became ready on port {port}"); + (state, port, env) +} + +/// The full Layer-2 acceptance path: a running serve instance holds the +/// repo's registration; `remove_from_index` (the CLI code path) must +/// DELEGATE to it (health probe → DELETE /repos/{alias}), serve must stop +/// holders and delete the DB directory WITHOUT being stopped, repos.json +/// must lose the entry, and a later query for the alias must be a clean +/// "Unknown alias" (no zombie stores) — all without stopping serve. +#[tokio::test] +#[serial_test::serial] +async fn index_rm_delegates_to_running_serve_end_to_end() { + let tmp = tempfile::tempdir().unwrap(); + let raw_proj = tmp.path().join("e2eproj"); + std::fs::create_dir(&raw_proj).unwrap(); + let proj = crate::cache::safe_canonicalize(&raw_proj).unwrap(); + // A real (empty) DB directory — stand-in for the LMDB dir; deleting it + // exercises the same remove_dir_all path without opening a real env. + std::fs::create_dir(proj.join(".codesearch.db")).unwrap(); + + let (state, _port, _env) = spawn_rm_delegation_test_serve(tmp.path(), |cfg| { + cfg.register(proj.clone()); + }) + .await; + // Capture the alias the registration derived (directory-name based) so + // the post-removal probe below addresses exactly what was registered. + let alias = ReposConfig::load() + .expect("seeded repos.json reload") + .repos + .iter() + .find(|(_, p)| **p == proj) + .map(|(a, _)| a.clone()) + .expect("seeded config must contain the proj"); + + // The CLI path — must delegate (serve is reachable) and succeed. + crate::index::remove_from_index(Some(proj.clone()), false) + .await + .expect("delegated index rm must succeed"); + + // DB directory gone WITHOUT stopping serve. + assert!( + !proj.join(".codesearch.db").exists(), + "serve must delete the DB dir during delegation — no serve stop needed" + ); + // repos.json (the shared temp file) lost the entry. + let after = ReposConfig::load().expect("repos.json reload after rm"); + assert!( + !after.repos.values().any(|p| p == &proj), + "entry must be unregistered from repos.json, still has: {:?}", + after.repos.values().collect::>() + ); + + // Later queries for the alias: a clean "Unknown alias", not a zombie + // store resurrecting the repo (todo #48 L2 acceptance criterion 3). + let err = state + .remove_repo(&alias) + .await + .expect_err("removing an unregistered alias must fail"); + assert!( + err.to_string().contains("Unknown alias"), + "expected a clean Unknown-alias error, got: {err:#}" + ); +} + +/// The hard variant of the Layer-2 acceptance ("file-delete succeeds without +/// serve-stop"): the running serve does not merely KNOW the repo — it holds a +/// REAL open LMDB environment on the `.codesearch.db` directory (a Warm +/// `RepoState`, exactly what a live query leaves behind). `remove_from_index` +/// must still delete the directory in one shot via delegation, without +/// stopping serve. The eviction is pinned from both sides: the registry +/// provably held a live env BEFORE the removal (precondition assert), and the +/// repos-map entry is gone AFTER it — while a holder is live the mmap'd +/// data/lock files cannot be deleted on Windows, so the dir-gone assert and +/// the eviction assert together prove the mechanism the locked-delete path +/// depends on, cross-platform. +/// +/// The store is opened via the production `try_open_stores` path (creating +/// the DB dir for real), and the returned `Arc` is MOVED into +/// `RepoState::Warm` with no clone kept — the test process must not itself +/// be the extra holder that defeats the delete. +#[tokio::test] +#[serial_test::serial] +async fn index_rm_deletes_db_while_serve_holds_real_lmdb_env() { + let tmp = tempfile::tempdir().unwrap(); + let raw_proj = tmp.path().join("heldenv"); + std::fs::create_dir(&raw_proj).unwrap(); + let proj = crate::cache::safe_canonicalize(&raw_proj).unwrap(); + let db_path = proj.join(DB_DIR_NAME); + // No pre-created DB dir: try_open_stores must create it, proving the + // env we then hold is a real production-shaped store. + + let (state, _port, _env) = spawn_rm_delegation_test_serve(tmp.path(), |cfg| { + cfg.register_with_alias(proj.clone(), Some("heldenv".to_string())) + .expect("seed registration must succeed"); + }) + .await; + + // Serve opens the repo FOR REAL — a live LMDB env under db_path. + let opened = state + .try_open_stores("heldenv", &db_path, true, false, None) + .expect("opening a real store for a brand-new repo must succeed"); + let OpenedStores::Write(stores) = opened else { + panic!("brand-new repo must open Write, not Readonly"); + }; + // Move the Arc in; keep NO clone (a test-side clone would be exactly the + // transient holder class remove_repo's retry has to out-wait). + state + .repos + .insert("heldenv".to_string(), RepoState::Warm { stores }); + + // Precondition: the env is genuinely held — this is what makes the + // delete impossible on Windows until serve's eviction releases it. + assert!( + !crate::lmdb_registry::open_holders_under(&db_path).is_empty(), + "test precondition: a real LMDB holder must be live under the db dir" + ); + + // The CLI path — delegated removal against a serve that HOLDS the env. + crate::index::remove_from_index(Some(proj.clone()), false) + .await + .expect("delegated index rm must succeed while serve holds the env"); + + // The DB directory is gone — deleted BY SERVE, in-place, serve still up. + assert!( + !db_path.exists(), + "serve must delete the really-held DB dir without being stopped" + ); + // The eviction genuinely dropped the RepoState (and with it the last + // Arc). NOTE: a registry query (`open_holders_under`) is + // VACUOUS here — the dir is deleted, canonicalize fails, and the helper + // by design answers "no holders" for a missing path even though a zombie + // env would still be alive on it (mutation-verified: skipping + // repos.remove left the test green through that assert on Linux). The + // repos-map assert is the non-vacuous pin: the Warm entry must be gone. + // On Windows a skipped eviction additionally fails the dir-gone assert + // (the mmap'd files refuse deletion while held). + assert!( + !state.repos.contains_key("heldenv"), + "eviction must drop the RepoState — a surviving Warm entry is a zombie holder" + ); + // repos.json lost the entry. + let after = ReposConfig::load().expect("repos.json reload after rm"); + assert!( + !after.repos.values().any(|p| p == &proj), + "entry must be unregistered from repos.json, still has: {:?}", + after.repos.values().collect::>() + ); + // No zombie: the alias no longer resolves for later queries. + let err = state + .remove_repo("heldenv") + .await + .expect_err("removing an unregistered alias must fail"); + assert!( + err.to_string().contains("Unknown alias"), + "expected a clean Unknown-alias error, got: {err:#}" + ); +} + +/// A cached C# symbol-index Error must NOT outlive the repo it belongs to. +/// +/// `repo_statuses_lightweight()` prefers the cached entry over its on-disk +/// probe, so an Error left behind by idle eviction renders a red `C#!` in the +/// TUI forever — a closed repo has no watcher left to retry a rebuild and +/// flip the state to Ready. Regression guard for the frozen-`!` fix observed +/// on a repo whose rebuild lost a one-shot LMDB double-open race days +/// earlier. +#[serial_test::serial] +#[test] +fn evicting_idle_repo_clears_frozen_csharp_error_state() { + let _env = crate::testing::EnvRestore::set(&[(crate::constants::REPO_IDLE_TIMEOUT_ENV, "1")]); + let state = ServeState::new(ReposConfig::default(), None); + + // Simulate the poisoned state: an Error + message cached by a failed + // watcher rebuild, and a last-access old enough to be evicted. + state + .csharp_index_status + .insert("frozen".to_string(), CSharpIndexStatus::Error); + state.csharp_index_error.insert( + "frozen".to_string(), + "LMDB double-open prevented".to_string(), + ); + state.last_access.insert( + "frozen".to_string(), + std::time::Instant::now() - std::time::Duration::from_secs(5), + ); + + state.evict_idle_repos(); + + assert!( + !state.csharp_index_status.contains_key("frozen"), + "eviction must clear the cached C# status — a frozen Error renders red forever otherwise" + ); + assert!( + !state.csharp_index_error.contains_key("frozen"), + "eviction must clear the cached C# error message along with the status" + ); +} + +/// The model a serve query is embedded with is read from the routed repo's own +/// index metadata — never assumed to be the hub-wide default. +/// +/// Regression guard for the serve hub pinning `ModelType::default()` (384-dim +/// MiniLM) for every query: on a hub whose indexes were rebuilt with +/// EmbeddingGemma that failed with "Query embedding dimension mismatch: +/// expected 768, got 384". Reintroducing the default pin makes the gemma cases +/// below fail. +#[test] +fn model_for_alias_reads_the_index_metadata_model() { + let cases = [ + ( + "embeddinggemma-q4", + Some(crate::embed::ModelType::EmbeddingGemma300MQ4), + ), + ("minilm-l6-q", Some(crate::embed::ModelType::AllMiniLML6V2Q)), + ("bge-base", Some(crate::embed::ModelType::BGEBaseENV15)), + // An unknown recorded name must not be silently coerced to the default: + // callers fall back explicitly, and the resolver reports "no answer". + ("not-a-real-model", None), + ]; + + for (model_short_name, expected) in cases { + let (_tmp, repo_path, state) = state_with_repo("repo"); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write( + db_path.join("metadata.json"), + format!(r#"{{"model_short_name":"{model_short_name}","dimensions":768}}"#), + ) + .unwrap(); + + assert_eq!( + state.model_for_alias("repo"), + expected, + "metadata model_short_name '{model_short_name}' must drive the query model" + ); + } +} + +/// Missing metadata (unindexed / legacy index) yields `None`, so the caller's +/// documented fallback to the default applies — and an unknown alias cannot +/// borrow another repo's model. +#[test] +fn model_for_alias_is_none_without_index_metadata() { + let (_tmp, _repo_path, state) = state_with_repo("repo"); + assert_eq!(state.model_for_alias("repo"), None); + assert_eq!(state.model_for_alias("not-registered"), None); +} + +/// A single hub can hold indexes built with different models: each alias +/// resolves independently, so a group fan-out embeds each store's query with +/// that store's own model. +#[test] +fn model_for_alias_is_per_repo_not_hub_wide() { + let tmp = tempfile::tempdir().unwrap(); + let config_file = tmp.path().join("repos.json"); + let mut config = ReposConfig::default(); + for (alias, model) in [("legacy", "minilm-l6-q"), ("rebuilt", "embeddinggemma-q4")] { + let repo_path = tmp.path().join(alias); + std::fs::create_dir(&repo_path).unwrap(); + config + .register_with_alias(repo_path.clone(), Some(alias.to_string())) + .unwrap(); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write( + db_path.join("metadata.json"), + format!(r#"{{"model_short_name":"{model}"}}"#), + ) + .unwrap(); + } + config.save_to(&config_file).unwrap(); + let state = ServeState::new(config, Some(config_file)); + + assert_eq!( + state.model_for_alias("legacy"), + Some(crate::embed::ModelType::AllMiniLML6V2Q) + ); + assert_eq!( + state.model_for_alias("rebuilt"), + Some(crate::embed::ModelType::EmbeddingGemma300MQ4) + ); +} + +/// The serve MCP service resolves the query model through the routed repo, not +/// its own (default) field. This is the exact seam the hub got wrong: it is the +/// service, not `ServeState`, that hands the model to the embedder. +#[test] +fn serve_service_uses_repo_model_not_default() { + let (_tmp, repo_path, state) = state_with_repo("gemma-repo"); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write( + db_path.join("metadata.json"), + r#"{"model_short_name":"embeddinggemma-q4","dimensions":768}"#, + ) + .unwrap(); + + let svc = crate::mcp::CodesearchService::new_for_serve(std::sync::Arc::new(state)).unwrap(); + + assert_eq!( + svc.query_model(Some("gemma-repo")), + crate::embed::ModelType::EmbeddingGemma300MQ4, + "serve must embed a repo's queries with the model that repo was indexed with" + ); + // No alias (unscoped) or an unknown alias falls back to the service default. + assert_eq!(svc.query_model(None), crate::embed::ModelType::default()); + assert_eq!( + svc.query_model(Some("not-registered")), + crate::embed::ModelType::default() + ); +} + +/// The grouped `status` model label must reflect the members' recorded models, +/// not the service default: a same-model group names that model, a mixed-model +/// hub says `mixed`. Regression guard for the status field reporting the +/// hardcoded default (`minilm-l6-q`) for every repo. +#[test] +fn group_status_model_label_is_common_or_mixed() { + let tmp = tempfile::tempdir().unwrap(); + let config_file = tmp.path().join("repos.json"); + let mut config = ReposConfig::default(); + for (alias, model) in [("legacy", "minilm-l6-q"), ("rebuilt", "embeddinggemma-q4")] { + let repo_path = tmp.path().join(alias); + std::fs::create_dir(&repo_path).unwrap(); + config + .register_with_alias(repo_path.clone(), Some(alias.to_string())) + .unwrap(); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write( + db_path.join("metadata.json"), + format!(r#"{{"model_short_name":"{model}"}}"#), + ) + .unwrap(); + } + config.save_to(&config_file).unwrap(); + let state = std::sync::Arc::new(ServeState::new(config, Some(config_file))); + let svc = crate::mcp::CodesearchService::new_for_serve(state).unwrap(); + + assert_eq!( + svc.group_model_label(&["legacy".to_string(), "rebuilt".to_string()]), + "mixed", + "a hub holding indexes built with different models must report 'mixed'" + ); + assert_eq!( + svc.group_model_label(&["rebuilt".to_string()]), + "embeddinggemma-q4", + "a single-model group must name its base model" + ); +} + +/// A fresh repo added with a model override must open its store at that model's +/// dimension, not the 384-dim default. Regression guard for `POST /repos` with +/// `model=embeddinggemma-q4`: the store used to be created at 384 and the +/// override applied only to metadata, so the reindex embedded 768-dim vectors +/// into a 384-dim store and indexed nothing. +#[tokio::test] +async fn try_open_stores_honours_dimension_override_for_a_fresh_repo() { + let tmp = tempfile::tempdir().unwrap(); + let repo_path = tmp.path().join("gemmarepo"); + std::fs::create_dir(&repo_path).unwrap(); + let db_path = repo_path.join(DB_DIR_NAME); + assert!(!db_path.exists(), "precondition: db dir must not exist yet"); + + let state = state_with_config(ReposConfig::default()); + + let stores = match state.try_open_stores("gemmarepo", &db_path, true, false, Some(768)) { + Ok(OpenedStores::Write(s)) => s, + Ok(OpenedStores::Readonly(_)) => panic!("expected Write, got Readonly"), + Err(e) => panic!("fresh open with a dimension override must succeed, got: {e}"), + }; + + let dims = stores + .vector_store + .read() + .await + .stats() + .expect("stats on a freshly created store") + .dimensions; + assert_eq!( + dims, 768, + "a repo added with --model embeddinggemma-q4 must open at 768 dims, not the 384 default" + ); +} + +#[test] +fn is_lmdb_format_corruption_matches_known_lmdb_errors() { + let cases = [ + ( + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + true, + ), + ("Symbol rebuild failed: heed -> MDB_BAD_VALSIZE", true), + ("storage error: wrong DUPFIXED size", true), + ("Unsupported size of key while opening vectordb", true), + ("scip-csharp failed with exit code 1", false), + ("MDB_NOTFOUND: No matching key/data pair found", false), + ("Task panicked: workspace load failed", false), + ("", false), + ]; + for (msg, expected) in cases { + assert_eq!( + ServeState::is_lmdb_format_corruption(msg), + expected, + "unexpected classification for {msg:?}" + ); + } +} + +#[tokio::test] +async fn enqueue_format_recovery_dedupes_and_starts_single_worker() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + // Same alias three times must collapse to one queued entry. The recovery + // worker is spawned but (current-thread test runtime) does not execute + // until an await point, so the queue content is asserted deterministically. + state.enqueue_format_recovery("ghost-repo"); + state.enqueue_format_recovery("ghost-repo"); + state.enqueue_format_recovery("ghost-repo"); + let len = state + .format_recovery_queue + .lock() + .expect("queue lock") + .len(); + assert_eq!(len, 1, "duplicate enqueues must collapse to one entry"); + assert!( + state + .format_recovery_worker_started + .load(std::sync::atomic::Ordering::Acquire), + "the first enqueue must start the recovery worker" + ); +} diff --git a/src/serve/tui.rs b/src/serve/tui.rs index de7ea5d8..9d31e60e 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -1060,7 +1060,7 @@ fn with_remote_stats(overlay: OverlayState, stats: RemoteStatsState) -> OverlayS /// Outcome of a TUI force-reindex launch — used to drive immediate footer /// feedback. Only describes whether the background task *started*; the actual /// indexing result is reported later via the status column / logs. -enum ReindexLaunch { +pub(crate) enum ReindexLaunch { /// Background reindex task spawned successfully. Started, /// A reindex was already running for this alias — request ignored. @@ -1071,7 +1071,7 @@ enum ReindexLaunch { /// Spawn a background force reindex task for the given repo alias. /// Follows the same flow as the HTTP `reindex_handler`. -fn spawn_force_reindex(alias: String, state: &Arc) -> ReindexLaunch { +pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> ReindexLaunch { // Guard against concurrent reindex if !state.begin_indexing(&alias) { tracing::warn!( @@ -1120,7 +1120,7 @@ fn spawn_force_reindex(alias: String, state: &Arc) -> ReindexLaunch None => { // Try to open stores (allow_create=true for recovery) let cancel = CancellationToken::new(); - match state.try_open_stores(&alias, &db_path, true, false) { + match state.try_open_stores(&alias, &db_path, true, false, None) { Ok(super::OpenedStores::Write(s)) => { state.repos.insert( alias.clone(), diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 52c04228..cd58d2f4 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -9,7 +9,7 @@ //! `rebuild()` calls `scip-csharp index` which now emits **definitions only** //! (no `FindReferencesAsync` loop). This makes a full rebuild 10–50× faster. //! -//! `find_references()` resolves references on demand: +//! `find_references_for_key()` resolves references on demand: //! 1. Return definitions from `scip_symbols` (always populated after rebuild). //! 2. Check `scip_ref_cache` for previously resolved references — return if present. //! 3. Cache miss: invoke `scip-csharp find-refs` for the single requested symbol, @@ -36,19 +36,70 @@ use std::collections::{HashMap, HashSet}; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::Arc; use std::thread; use std::time::{SystemTime, UNIX_EPOCH}; use crate::lmdb_registry::TrackedEnv; use anyhow::{bail, Context, Result}; use heed::types::{Bytes, Str}; -use heed::{Database, EnvOpenOptions}; +use heed::Database; use serde::{Deserialize, Serialize}; use super::scip_parse; -use super::{PrewarmSummary, RebuildScope, RebuildSummary, SymbolIndexer, SymbolReference}; +use super::{ + ImpactQuery, KeyMatch, PrewarmSummary, RebuildScope, RebuildSummary, SymbolIndexer, + SymbolReference, +}; + +// ── Helper stderr routing ───────────────────────────────────────── + +/// True when a scip-csharp stderr line carries warning-severity output: +/// the helper's own `[WARN]` prefix or MSBuildWorkspace's `[Failure]` +/// workspace diagnostics (e.g. project-load failures). +pub(crate) fn is_helper_warning_line(line: &str) -> bool { + line.contains("[WARN]") || line.contains("[Failure]") +} -use crate::constants::{SCIP_LMDB_DEFAULT_MAP_SIZE_MB, SCIP_LMDB_MAP_SIZE_MB_ENV}; +/// Route one scip-csharp stderr line into tracing at the right severity. +/// This is the ONLY sanctioned path for helper stderr: spawn helpers with +/// `Stdio::piped()` and drain through here — never `Stdio::inherit()`, +/// which bypasses tracing entirely and sprays raw MSBuild output over the +/// serve process (file-only logging, TUI on stderr), scrambling it. +pub(crate) fn emit_helper_stderr_line(tag: &str, label: &str, line: &str) { + if is_helper_warning_line(line) { + tracing::warn!("[{tag}:{label}] {line}"); + } else { + tracing::info!("[{tag}:{label}] {line}"); + } +} + +/// Drain a helper output pipe to EOF, invoking `emit` once per line. +/// Undecodable bytes are lossy-decoded and the line is still emitted, and +/// a persistent read error ends the drain. A drain must never stop on +/// individual bad lines: a stalled drain lets the pipe fill and blocks the +/// helper mid-workspace-load (`lines().map_while(Result::ok)` had exactly +/// that failure mode; `filter_map(Result::ok)` traded it for a busy spin +/// under `clippy::lines_filter_map_ok` — read_until has neither problem). +pub(crate) fn drain_pipe_to_tracing(pipe: R, mut emit: impl FnMut(&str)) { + let mut reader = BufReader::new(pipe); + loop { + let mut buf = Vec::new(); + match reader.read_until(b'\n', &mut buf) { + Ok(0) => break, // EOF — helper died; drain thread exits here + Ok(_) => { + while matches!(buf.last(), Some(b'\n') | Some(b'\r')) { + buf.pop(); + } + emit(&String::from_utf8_lossy(&buf)); + } + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => { + continue; // transient (EINTR) — retry the read, do NOT end the drain + } + Err(_) => break, // dead pipe — nothing more to drain + } + } +} // ── Constants ───────────────────────────────────────────────────── @@ -56,7 +107,7 @@ use crate::constants::{SCIP_LMDB_DEFAULT_MAP_SIZE_MB, SCIP_LMDB_MAP_SIZE_MB_ENV} const SCIP_DB_NAME: &str = crate::constants::SCIP_SYMBOLS_DB_NAME; /// LMDB database name for the rebuild timestamp. -const SCIP_META_DB_NAME: &str = "scip_meta"; +const SCIP_META_DB_NAME: &str = crate::constants::SCIP_META_DB_NAME; /// LMDB database name for the position-to-symbols index. const SCIP_POSITION_DB_NAME: &str = crate::constants::SCIP_POSITION_DB_NAME; @@ -67,9 +118,20 @@ const SCIP_SIMPLE_NAMES_DB_NAME: &str = crate::constants::SCIP_SIMPLE_NAMES_DB_N /// LMDB database name for the on-demand reference cache (populated by find-refs). const SCIP_REF_CACHE_DB_NAME: &str = crate::constants::SCIP_REF_CACHE_DB_NAME; +/// LMDB database name for per-symbol completeness warnings persisted +/// alongside the reference cache (absence of an entry = complete). +const SCIP_REF_WARNINGS_DB_NAME: &str = crate::constants::SCIP_REF_WARNINGS_DB_NAME; + /// Key in the meta database that stores the last rebuild timestamp (UNIX epoch seconds). const META_REBUILD_TS: &str = crate::constants::SCIP_REBUILD_TIMESTAMP_KEY; +/// Key in the meta database storing the git HEAD sha the index was built for. +const META_HEAD_SHA: &str = crate::constants::SCIP_HEAD_SHA_KEY; + +/// Key in the meta database recording the key-format generation the index was +/// built with (see [`crate::constants::SCIP_KEY_FORMAT`]). +const META_KEY_FORMAT: &str = crate::constants::SCIP_KEY_FORMAT_KEY; + /// Key in the meta database storing the count of indexed symbols. #[allow(dead_code)] const META_SYMBOL_COUNT: &str = "symbol_count"; @@ -175,6 +237,44 @@ fn deserialize_keys_v1(bytes: &[u8]) -> Result> { bincode::deserialize(&bytes[1..]).with_context(|| "bincode deserialize keys failed") } +// ── Ref-resolution warnings persistence ─────────────────────────── + +/// Persist one canonical key's resolution warnings into `scip_ref_warnings`, +/// in the caller's transaction so the cached refs and their honesty land as +/// ONE atomic fact. Empty warnings REMOVE the entry — absence means +/// "complete", so a later clean re-resolution clears a stale warning +/// instead of reporting it forever. (Wire format = the key-list format: +/// version byte + bincode Vec.) +fn store_ref_warnings( + env: &TrackedEnv, + wtxn: &mut heed::RwTxn<'_>, + canonical: &str, + warnings: &[String], +) -> Result<()> { + let db: Database = env.create_database(wtxn, Some(SCIP_REF_WARNINGS_DB_NAME))?; + if warnings.is_empty() { + db.delete(wtxn, canonical)?; + } else { + let bytes = serialize_keys_v1(warnings) + .with_context(|| format!("Failed to serialize warnings for {canonical}"))?; + db.put(wtxn, canonical, &bytes)?; + } + Ok(()) +} + +/// Read one canonical key's warnings. Missing database or entry — and an +/// undecodable value — read as empty (complete): a warning that cannot be +/// read must not fail a lookup that has valid references. +fn read_ref_warnings(env: &TrackedEnv, rtxn: &heed::RoTxn<'_>, canonical: &str) -> Vec { + match env.open_database::(rtxn, Some(SCIP_REF_WARNINGS_DB_NAME)) { + Ok(Some(db)) => match db.get(rtxn, canonical) { + Ok(Some(bytes)) => deserialize_keys_v1(bytes).unwrap_or_default(), + _ => Vec::new(), + }, + _ => Vec::new(), + } +} + // ── Simple-name extraction ───────────────────────────────────────── /// Extracts the last segment of a canonical SCIP symbol as a simple name. @@ -390,44 +490,15 @@ impl CSharpSymbolIndexer { None } - /// Open or create the SCIP LMDB environment for a given repo database path. + /// Open the shared SCIP LMDB environment for a given repo database path. /// - /// Pre-opens ALL named databases so they exist before first use. - /// LMDB requires named DBs to be created (or opened) in a write txn - /// before they can be read in later read txns within the same env session. - fn open_scip_env(&self, db_path: &Path) -> Result { - let scip_dir = db_path.join("scip"); - std::fs::create_dir_all(&scip_dir) - .with_context(|| format!("Failed to create SCIP directory: {}", scip_dir.display()))?; - - // SAFETY: same pattern as vectordb/store.rs — LMDB mmap contract. - // TrackedEnv additionally prevents double-open within the same process. - // - // map_size is virtual address space (not RSS). 512 MB default is safe on - // both Windows and POSIX; the OS only faults in pages that are written. - // Enterprise repos with thousands of symbols + Phase-3 ref_cache can - // exceed the old 64 MB limit, causing MDB_MAP_FULL on cache writes. - let map_size_mb = std::env::var(SCIP_LMDB_MAP_SIZE_MB_ENV) - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(SCIP_LMDB_DEFAULT_MAP_SIZE_MB); - let mut opts = EnvOpenOptions::new(); - opts.map_size(map_size_mb * 1024 * 1024).max_dbs(10); - // SAFETY: `NO_TLS` only changes reader-slot tracking. See `BASE_ENV_FLAGS`. - unsafe { opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS) }; - let env = - unsafe { TrackedEnv::open(&opts, &scip_dir, &format!("SCIP({})", db_path.display()))? }; - - // Eagerly create / re-open all named databases. - let mut wtxn = env.write_txn()?; - env.create_database::(&mut wtxn, Some(SCIP_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_META_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_POSITION_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_SIMPLE_NAMES_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_REF_CACHE_DB_NAME))?; - wtxn.commit()?; - - Ok(env) + /// Delegates to [`crate::symbols::get_shared_scip_env`]: one environment + /// per `db_path/scip` for the whole process, shared across concurrent + /// queries, rebuilds and the TypeScript adapter, so overlapping users + /// serialise on LMDB's writer mutex instead of failing the double-open + /// guard. + fn open_scip_env(&self, db_path: &Path) -> Result> { + crate::symbols::get_shared_scip_env(db_path) } // ── Helper invocation ────────────────────────────────────────── @@ -467,22 +538,22 @@ impl CSharpSymbolIndexer { let stderr_handle = child.stderr.take().map(|stderr| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stderr).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stderr, |line| { if !line.is_empty() { - tracing::info!("[scip-csharp:{}] {}", label, line); + emit_helper_stderr_line("scip-csharp", &label, line); } - } + }); }) }); let stdout_handle = child.stdout.take().map(|stdout| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stdout).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stdout, |line| { if !line.is_empty() { tracing::debug!("[scip-csharp:{}] {}", label, line); } - } + }); }) }); @@ -498,23 +569,28 @@ impl CSharpSymbolIndexer { } if !status.success() { - tracing::warn!("scip-csharp exited with {} for {}", status, solution_short); + tracing::warn!( + "scip-csharp exited with {} for {}", + super::exit_status_text(&status), + solution_short + ); // Don't bail — partial output is acceptable per AGENTS.md spec } Ok(()) } - /// Invoke `scip-csharp find-refs` for a single symbol and return its references. + /// Invoke `scip-csharp find-refs` for a single symbol and return its + /// references plus the completeness warnings the helper reported. /// - /// This is the "lazy" half of Opt 2: called on first `find_impact` for a symbol - /// that has not yet been resolved. Result is cached in `scip_ref_cache`. + /// This is the "lazy" half of Opt 2: called on first `find_impact` for a + /// symbol that has not yet been resolved. Result is cached in `scip_ref_cache`. fn invoke_find_refs_helper( &self, helper: &Path, solution: &Path, symbol: &str, - ) -> Result> { + ) -> Result<(Vec, Vec)> { let start = std::time::Instant::now(); let temp_dir = std::env::temp_dir().join("codesearch-scip"); @@ -559,22 +635,22 @@ impl CSharpSymbolIndexer { let stderr_handle = child.stderr.take().map(|stderr| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stderr).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stderr, |line| { if !line.is_empty() { - tracing::info!("[scip-csharp find-refs:{}] {}", label, line); + emit_helper_stderr_line("scip-csharp find-refs", &label, line); } - } + }); }) }); let stdout_handle = child.stdout.take().map(|stdout| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stdout).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stdout, |line| { if !line.is_empty() { tracing::debug!("[scip-csharp find-refs:{}] {}", label, line); } - } + }); }) }); @@ -592,7 +668,7 @@ impl CSharpSymbolIndexer { if !status.success() { tracing::warn!( "scip-csharp find-refs exited with {} for '{}'", - status, + super::exit_status_text(&status), symbol ); } @@ -624,46 +700,56 @@ impl CSharpSymbolIndexer { start.elapsed().as_millis() ); - Ok(stored) + Ok((stored, result.warnings)) } // ── Internal lookup helpers ──────────────────────────────────── - /// Resolve a (possibly fuzzy) symbol name to the canonical SCIP key stored - /// in `scip_symbols`. Returns `None` if no matching symbol is found. - fn resolve_canonical_key(&self, env: &TrackedEnv, symbol: &str) -> Result> { + /// Resolve a (possibly fuzzy) symbol name to canonical SCIP key(s). + /// + /// Exact key wins. Otherwise the simple-name index lists candidates and + /// the fuzzy filter narrows them: zero matches is `NotFound`, exactly + /// one resolves, and SEVERAL come back as `KeyMatch::Ambiguous` — the + /// caller must choose. The old behaviour silently picked the shortest + /// candidate, which hid overloads (`Validate()` vs `Validate(string)`) + /// from the caller and answered about the wrong symbol. + fn resolve_name_key(&self, env: &TrackedEnv, symbol: &str) -> Result { let rtxn = env.read_txn()?; let symbols_db: Database = match env.open_database(&rtxn, Some(SCIP_DB_NAME))? { Some(db) => db, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; // Exact match first if symbols_db.get(&rtxn, symbol)?.is_some() { - return Ok(Some(symbol.to_string())); + return Ok(KeyMatch::Resolved(symbol.to_string())); } // Fuzzy via simple-name index let simple_names_db: Database = match env.open_database(&rtxn, Some(SCIP_SIMPLE_NAMES_DB_NAME))? { Some(db) => db, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; let simple = extract_simple_name(symbol); let candidates: Vec = match simple_names_db.get(&rtxn, &simple as &str)? { Some(b) => deserialize_keys_v1(b)?, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; - let chosen = candidates - .iter() + let mut matches: Vec = candidates + .into_iter() .filter(|k| fuzzy_symbol_match(symbol, k)) - .min_by_key(|k| k.len()) - .cloned(); - - Ok(chosen) + .collect(); + matches.sort(); + matches.dedup(); + Ok(match matches.len() { + 0 => KeyMatch::NotFound, + 1 => KeyMatch::Resolved(matches.pop().expect("len checked")), + _ => KeyMatch::Ambiguous(matches), + }) } /// Inner implementation: fetch references for an EXACT (canonical) symbol key. @@ -674,9 +760,9 @@ impl CSharpSymbolIndexer { /// /// Inner implementation: fetch references for an EXACT (canonical) symbol key. /// - /// Opens its own LMDB environment so the caller's env handle (if any) is not - /// held concurrently with the internal write txn that caches lazy results. - /// This avoids the "two Env objects on the same path" footgun. + /// Uses the shared SCIP env ([`crate::symbols::get_shared_scip_env`]); the + /// internal write txn that caches lazy results serialises against other + /// writers on LMDB's single-writer mutex instead of erroring the loser. fn find_refs_for_canonical_key( &self, db_path: &Path, @@ -731,6 +817,16 @@ impl CSharpSymbolIndexer { } // rtxn dropped here if cache_hit || has_legacy_refs { + // A cached answer replays the warnings persisted WITH it — the + // honesty is part of the cached fact, so a partial result can + // never quietly pass for complete on the 2nd+ call. + let warnings = { + let rtxn = env.read_txn()?; + read_ref_warnings(&env, &rtxn, canonical) + }; + for w in &warnings { + tracing::warn!("cached refs for '{}' may be incomplete: {}", canonical, w); + } return Ok(all_stored.into_iter().map(stored_to_symbol_ref).collect()); } @@ -783,7 +879,39 @@ impl CSharpSymbolIndexer { canonical ); - let lazy_refs = self.invoke_find_refs_helper(&helper, &solution, canonical)?; + // Preferred path: the resident workspace pool (todo #115) — the + // solution's Roslyn workspace stays loaded for MAX_RESIDENT repos, + // so after the first lookup this answers in seconds instead of + // spawning a fresh helper per call. Fallback: the one-shot spawn, + // which keeps working when the pool cannot (spawn failure, heap-cap + // death, eviction race) — correctness never depends on residency. + // Both paths carry completeness warnings; a partial answer must be + // cached AS partial, never as a complete one. + let (lazy_refs, lazy_warnings): (Vec, Vec) = + match crate::symbols::resident::WORKSPACE_POOL.find_refs(&helper, &solution, canonical) + { + Ok(resident) => ( + resident + .references + .into_iter() + .map(|r| StoredReference { + file: r.file, + start_line: r.start_line, + end_line: r.end_line, + kind: r.kind, + }) + .collect(), + resident.warnings, + ), + Err(e) => { + tracing::warn!( + "resident helper unavailable ({e:#}); falling back to one-shot \ + find-refs for '{}'", + canonical + ); + self.invoke_find_refs_helper(&helper, &solution, canonical)? + } + }; // ── Write phase — cache the resolved references ──────────── { @@ -793,6 +921,9 @@ impl CSharpSymbolIndexer { let cached_bytes = serialize_refs(&lazy_refs) .with_context(|| format!("Failed to serialize refs for cache: {}", canonical))?; ref_cache_db.put(&mut wtxn, canonical, &cached_bytes)?; + // Same txn as the refs: cached-partial and its warnings are one + // atomic fact. Empty warnings remove any stale entry. + store_ref_warnings(&env, &mut wtxn, canonical, &lazy_warnings)?; wtxn.commit()?; } @@ -992,22 +1123,22 @@ impl CSharpSymbolIndexer { let stderr_handle = child.stderr.take().map(|stderr| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stderr).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stderr, |line| { if !line.is_empty() { - tracing::info!("[scip-csharp batch-find-refs:{}] {}", label, line); + emit_helper_stderr_line("scip-csharp batch-find-refs", &label, line); } - } + }); }) }); let stdout_handle = child.stdout.take().map(|stdout| { let label = solution_short.clone(); thread::spawn(move || { - for line in BufReader::new(stdout).lines().map_while(Result::ok) { + drain_pipe_to_tracing(stdout, |line| { if !line.is_empty() { tracing::debug!("[scip-csharp batch-find-refs:{}] {}", label, line); } - } + }); }) }); @@ -1025,7 +1156,7 @@ impl CSharpSymbolIndexer { if !status.success() { tracing::warn!( "scip-csharp batch-find-refs exited with {} for {}", - status, + super::exit_status_text(&status), solution_short ); // Don't bail — partial output is acceptable @@ -1055,6 +1186,10 @@ impl CSharpSymbolIndexer { struct SymbolResult { symbol: String, references: Vec, + /// Absent in helper output from before warnings existed — + /// default to empty so old binaries keep parsing as "complete". + #[serde(default)] + warnings: Vec, } #[derive(serde::Deserialize)] @@ -1106,6 +1241,9 @@ impl CSharpSymbolIndexer { let bytes = serialize_refs(&refs) .with_context(|| format!("Failed to serialize batch refs for {}", result.symbol))?; ref_cache_db.put(&mut wtxn, result.symbol.as_str(), &bytes)?; + // Same txn as the refs: cached-partial and its warnings are one + // atomic fact. Empty warnings remove any stale entry. + store_ref_warnings(&env, &mut wtxn, &result.symbol, &result.warnings)?; cached_count += 1; } @@ -1510,6 +1648,20 @@ impl SymbolIndexer for CSharpSymbolIndexer { META_REPO_PATH, repo_path.to_string_lossy().as_ref(), )?; + // Fingerprint the index: which HEAD it was built for. Skipped when + // git is unreadable (better absent than a wrong claim); an older + // value from a previous build may then survive, which stays + // approximately right for the usual same-branch rebuild. + if let Some(sha) = super::current_git_head(repo_path) { + meta_db.put(&mut wtxn, META_HEAD_SHA, sha.as_str())?; + } + // Unconditional: has_index refuses an index whose key-format stamp is + // absent or stale, so a key-format change forces exactly one rebuild. + meta_db.put( + &mut wtxn, + META_KEY_FORMAT, + crate::constants::SCIP_KEY_FORMAT, + )?; wtxn.commit()?; @@ -1530,55 +1682,80 @@ impl SymbolIndexer for CSharpSymbolIndexer { }) } - fn find_references(&self, db_path: &Path, symbol: &str) -> Result> { - // Resolve to canonical key in a short-lived env scope, then drop it before - // entering find_refs_for_canonical_key (which opens its own env). - // This ensures no two Env handles are live on the same path simultaneously. - let canonical = { - let env = self.open_scip_env(db_path)?; - match self.resolve_canonical_key(&env, symbol)? { - Some(k) => k, - None => { - tracing::debug!("Symbol '{}' not found in index", symbol); - return Ok(vec![]); - } + fn resolve_query(&self, db_path: &Path, query: &ImpactQuery) -> Result { + match query { + ImpactQuery::ExactKey(key) => { + // Explicit selection: presence check only, no fuzzy + // fallback. A key that is not in the index is NotFound — + // the handler turns that into a loud failure, never a + // guess at a near-miss symbol. + let env = self.open_scip_env(db_path)?; + let rtxn = env.read_txn()?; + let present = match env.open_database::(&rtxn, Some(SCIP_DB_NAME))? { + Some(db) => db.get(&rtxn, key as &str)?.is_some(), + None => false, + }; + Ok(if present { + KeyMatch::Resolved(key.clone()) + } else { + KeyMatch::NotFound + }) } - // env dropped here - }; - - self.find_refs_for_canonical_key(db_path, &canonical) + ImpactQuery::Name(name) => { + let env = self.open_scip_env(db_path)?; + self.resolve_name_key(&env, name) + } + ImpactQuery::Position { file, line } => { + let env = self.open_scip_env(db_path)?; + let rtxn = env.read_txn()?; + + let positions_db: Database = env + .open_database(&rtxn, Some(SCIP_POSITION_DB_NAME))? + .ok_or_else(|| { + anyhow::anyhow!("Position index not found. Run a rebuild first.") + })?; + + // Normalize file path to forward-slash (Windows compat) + let pos_key = format!("{}:{}", file.to_string_lossy().replace('\\', "/"), line); + + let mut candidates: Vec = match positions_db.get(&rtxn, &pos_key as &str)? { + Some(b) => deserialize_keys_v1(b)?, + None => return Ok(KeyMatch::NotFound), + }; + candidates.sort(); + candidates.dedup(); + + // Several symbols on one line (overloads, partial spans) + // are ambiguity, not a licence to pick the shortest. + Ok(match candidates.len() { + 0 => KeyMatch::NotFound, + 1 => KeyMatch::Resolved(candidates.pop().expect("len checked")), + _ => KeyMatch::Ambiguous(candidates), + }) + } + } } - fn find_references_by_position( + fn find_references_for_key( &self, db_path: &Path, - file: &Path, - line: u32, + canonical_key: &str, ) -> Result> { - let env = self.open_scip_env(db_path)?; - let rtxn = env.read_txn()?; - - let positions_db: Database = env - .open_database(&rtxn, Some(SCIP_POSITION_DB_NAME))? - .ok_or_else(|| anyhow::anyhow!("Position index not found. Run a rebuild first."))?; - - // Normalize file path to forward-slash (Windows compat) - let pos_key = format!("{}:{}", file.to_string_lossy().replace('\\', "/"), line); + self.find_refs_for_canonical_key(db_path, canonical_key) + } - let candidate_keys: Vec = match positions_db.get(&rtxn, &pos_key as &str)? { - Some(b) => deserialize_keys_v1(b)?, - None => return Ok(vec![]), + fn lookup_warnings(&self, db_path: &Path, canonical: &str) -> Vec { + // Plain LMDB read — never a helper invocation, so the find_impact + // handler can call it after a lookup without risking minutes of work. + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return Vec::new(), }; - - // Pick shortest (most specific) symbol defined at this position - let chosen = candidate_keys.iter().min_by_key(|k| k.len()).cloned(); - drop(rtxn); - drop(env); // must drop before find_refs_for_canonical_key opens its own env - - match chosen { - Some(k) => self.find_refs_for_canonical_key(db_path, &k), - None => Ok(vec![]), - } + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return Vec::new(), + }; + read_ref_warnings(&env, &rtxn, canonical) } fn index_age(&self, db_path: &Path) -> u64 { @@ -1614,13 +1791,48 @@ impl SymbolIndexer for CSharpSymbolIndexer { now.saturating_sub(stored_ts) } + fn index_head_sha(&self, db_path: &Path) -> Option { + let env = self.open_scip_env(db_path).ok()?; + let rtxn = env.read_txn().ok()?; + let meta_db: Database = env + .open_database(&rtxn, Some(SCIP_META_DB_NAME)) + .ok() + .flatten()?; + let sha = meta_db.get(&rtxn, META_HEAD_SHA).ok().flatten()?; + let sha = sha.trim().to_string(); + (!sha.is_empty()).then_some(sha) + } + + /// Whether a SCIP index exists AND was built with the current key format. + /// An index that is fresh by timestamp but stamped with another (or no) + /// key-format generation would serve old-shaped canonical keys as truth, + /// so it reports as absent and the caller rebuilds. fn has_index(&self, db_path: &Path) -> bool { let scip_dir = db_path.join("scip"); if !scip_dir.exists() { return false; } // Quick check: if index_age is finite, the index exists - self.index_age(db_path) != u64::MAX + if self.index_age(db_path) == u64::MAX { + return false; + } + // Same env/txn/open pattern as `index_head_sha` above. + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return false, + }; + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return false, + }; + let meta_db: Option> = env + .open_database(&rtxn, Some(SCIP_META_DB_NAME)) + .ok() + .flatten(); + let stored = meta_db + .and_then(|db| db.get(&rtxn, META_KEY_FORMAT).ok().flatten()) + .map(|s| s.trim().to_string()); + stored.as_deref() == Some(crate::constants::SCIP_KEY_FORMAT) } fn is_available(&self) -> bool { @@ -1739,4 +1951,470 @@ mod tests { "csharp App . FieldDefinition#Validate()." )); } + + // ── has_index key-format gate (B4) ──────────────────────────────── + + /// A rebuild-stamped meta entry is what the gate reads; the fixture + /// hand-populates scip_meta exactly like `rebuild` does (timestamp + + /// key_format) instead of running a real rebuild (needs the helper). + #[test] + fn has_index_refuses_indexes_not_stamped_with_the_current_key_format() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let indexer = CSharpSymbolIndexer::new(); + + let put_meta = |key: &str, value: &str| { + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: Database = env + .open_database(&wtxn, Some(SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + meta.put(&mut wtxn, META_REBUILD_TS, "0").unwrap(); + if !key.is_empty() { + meta.put(&mut wtxn, key, value).unwrap(); + } + wtxn.commit().unwrap(); + }; + + // Pre-B4 shape: fresh timestamp, NO key_format meta → refused. + put_meta("", ""); + assert!( + !indexer.has_index(&db), + "an index without key_format meta must not count as an index" + ); + + // Current key format → accepted. + put_meta(META_KEY_FORMAT, crate::constants::SCIP_KEY_FORMAT); + assert!( + indexer.has_index(&db), + "an index stamped with the current key format must be accepted" + ); + + // A previous generation's stamp → refused again. + put_meta(META_KEY_FORMAT, "1"); + assert!( + !indexer.has_index(&db), + "an index stamped with an older key format must be rebuilt, not served" + ); + } + + // ── resolve_query semantics (hand-populated LMDB — no helper) ── + + /// Two `Validate` overloads share a simple name, `Compute` is unique. + /// Writes scip_symbols / scip_simple_names / scip_positions directly. + fn populate_ambiguity_fixture(db_path: &Path) -> (String, String, String) { + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + + let validate1 = "csharp Ns . V#Validate().".to_string(); + let validate2 = "csharp Ns . V#Validate(System.String).".to_string(); + let compute = "csharp Ns . C#Compute().".to_string(); + + let symbols: Database = env + .open_database(&wtxn, Some(SCIP_DB_NAME)) + .unwrap() + .unwrap(); + for key in [&validate1, &validate2, &compute] { + let refs = serialize_refs(&[StoredReference { + file: PathBuf::from("src/v.cs"), + start_line: 1, + end_line: 1, + kind: "definition".into(), + }]) + .unwrap(); + symbols.put(&mut wtxn, key.as_str(), &refs).unwrap(); + } + + let names: Database = env + .open_database(&wtxn, Some(SCIP_SIMPLE_NAMES_DB_NAME)) + .unwrap() + .unwrap(); + // Stored deliberately out of order: resolution must sort. + names + .put( + &mut wtxn, + "Validate", + &serialize_keys_v1(&[validate2.clone(), validate1.clone()]).unwrap(), + ) + .unwrap(); + names + .put( + &mut wtxn, + "Compute", + &serialize_keys_v1(std::slice::from_ref(&compute)).unwrap(), + ) + .unwrap(); + + let positions: Database = env + .open_database(&wtxn, Some(SCIP_POSITION_DB_NAME)) + .unwrap() + .unwrap(); + positions + .put( + &mut wtxn, + "src/v.cs:10", + &serialize_keys_v1(&[validate1.clone(), validate2.clone()]).unwrap(), + ) + .unwrap(); + positions + .put( + &mut wtxn, + "src/v.cs:20", + &serialize_keys_v1(std::slice::from_ref(&compute)).unwrap(), + ) + .unwrap(); + + wtxn.commit().unwrap(); + (validate1, validate2, compute) + } + + #[test] + fn resolve_name_unique_fuzzy_resolves_the_single_candidate() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (_v1, _v2, compute) = populate_ambiguity_fixture(&db); + let indexer = CSharpSymbolIndexer::new(); + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::Name("Compute".into())) + .unwrap(), + KeyMatch::Resolved(compute) + ); + } + + #[test] + fn resolve_name_overloads_come_back_ambiguous_and_sorted() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (v1, v2, _compute) = populate_ambiguity_fixture(&db); + let indexer = CSharpSymbolIndexer::new(); + let m = indexer + .resolve_query(&db, &ImpactQuery::Name("Validate".into())) + .unwrap(); + // The pre-fix behaviour silently picked the shortest key here and + // answered about the wrong overload. + assert_eq!(m, KeyMatch::Ambiguous(vec![v1, v2])); + } + + #[test] + fn resolve_exact_key_is_verbatim_and_never_fuzzy() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (v1, _v2, _compute) = populate_ambiguity_fixture(&db); + let indexer = CSharpSymbolIndexer::new(); + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::ExactKey(v1.clone())) + .unwrap(), + KeyMatch::Resolved(v1) + ); + // A key that is only a fuzzy neighbour of a stored one must miss. + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::ExactKey("csharp Ns . V#Validate".into())) + .unwrap(), + KeyMatch::NotFound + ); + } + + #[test] + fn resolve_position_single_resolves_two_symbols_are_ambiguous() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (v1, v2, compute) = populate_ambiguity_fixture(&db); + let indexer = CSharpSymbolIndexer::new(); + + // One symbol on the line → resolves. + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/v.cs"), + line: 20 + } + ) + .unwrap(), + KeyMatch::Resolved(compute) + ); + + // Two overloads on the same line → ambiguity, never a shortest pick. + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/v.cs"), + line: 10 + } + ) + .unwrap(), + KeyMatch::Ambiguous(vec![v1, v2]) + ); + + // Nothing defined there → NotFound. + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/v.cs"), + line: 99 + } + ) + .unwrap(), + KeyMatch::NotFound + ); + } + + #[test] + fn references_for_key_returns_stored_definitions() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (v1, _v2, _compute) = populate_ambiguity_fixture(&db); + let indexer = CSharpSymbolIndexer::new(); + // Safe without the helper: the fixture stores definitions, and both + // the no-helper and no-.sln lazy paths short-circuit to definitions. + let refs = indexer.find_references_for_key(&db, &v1).unwrap(); + assert_eq!(refs.len(), 1, "definitions only, got {refs:?}"); + assert_eq!(refs[0].kind, "definition"); + assert_eq!(refs[0].file, PathBuf::from("src/v.cs")); + } + + #[test] + fn lookup_warnings_reads_the_ref_warnings_db_and_absence_is_empty() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (v1, _v2, compute) = populate_ambiguity_fixture(&db); + + // Hand-populate warnings for v1 only (same wire format the lazy and + // batch write paths use: version byte + bincode Vec). + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + { + let mut wtxn = env.write_txn().unwrap(); + store_ref_warnings( + &env, + &mut wtxn, + &v1, + &[ + "FindReferencesAsync failed for Validate: InvalidOperationException: boom" + .to_string(), + ], + ) + .unwrap(); + wtxn.commit().unwrap(); + } + + let indexer = CSharpSymbolIndexer::new(); + let warnings = indexer.lookup_warnings(&db, &v1); + assert_eq!(warnings.len(), 1, "the stored warning must be read back"); + assert!( + warnings[0].contains("FindReferencesAsync failed"), + "warning text must round-trip, got: {}", + warnings[0] + ); + + // A key with no entry reads as empty (complete), never an error. + assert!(indexer.lookup_warnings(&db, &compute).is_empty()); + + // Empty warnings must REMOVE the entry: absence = complete, so a + // clean re-resolution clears a stale warning instead of reporting + // it forever (a partial find-refs result that was cached and later + // re-resolved cleanly must stop claiming partiality). + { + let mut wtxn = env.write_txn().unwrap(); + store_ref_warnings(&env, &mut wtxn, &v1, &[]).unwrap(); + wtxn.commit().unwrap(); + } + assert!( + indexer.lookup_warnings(&db, &v1).is_empty(), + "storing empty warnings must clear the persisted entry" + ); + } + + // ── B3 warnings: producer-side coverage (the sites that WRITE) ── + // + // Every test above hand-populates the warnings store (the consumer + // half). These two drive the real producer sites instead, so deleting + // the store_ref_warnings call in parse_and_cache_batch_refs or in the + // lazy write phase fails here. + + /// A real helper executable, built with the test run's own rustc. + /// + /// A `.cmd` script cannot stand in: `validate_helper_path` accepts only + /// the literal filename `scip-csharp(.exe)`, and CreateProcess refuses + /// batch content under an `.exe` name, so the file must be a real PE. + /// Behaviour: `serve` exits without the handshake (the pool's wait_ready + /// hits EOF and errors, routing the caller through the one-shot + /// fallback); `find-refs` writes a fixed warnings-bearing JSON to its + /// `--output` path, mirroring a helper that survived a partial failure. + const FAKE_HELPER_SRC: &str = r#"fn main() { + let args: Vec = std::env::args().collect(); + if args.get(1).map(String::as_str) == Some("serve") { + return; + } + let out = args + .iter() + .position(|a| a == "--output") + .and_then(|i| args.get(i + 1)) + .expect("usage: find-refs ... --output "); + let json = "{\"version\": \"2.0\", \"symbol\": \"csharp Ns . V#Validate().\", \"references\": [{\"file\": \"src/generated.cs\", \"start_line\": 11, \"end_line\": 11, \"kind\": \"reference\"}], \"warnings\": [\"FindReferencesAsync failed for Validate: InvalidOperationException: boom\"]}"; + std::fs::write(out, json).unwrap(); +} +"#; + + fn build_fake_csharp_helper(dir: &Path) -> PathBuf { + let src_path = dir.join("fake_helper.rs"); + std::fs::write(&src_path, FAKE_HELPER_SRC).unwrap(); + let exe_path = dir.join(if cfg!(windows) { + "scip-csharp.exe" + } else { + "scip-csharp" + }); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let status = std::process::Command::new(rustc) + .args(["--edition", "2021", "-Cdebuginfo=0"]) + .arg("-o") + .arg(&exe_path) + .arg(&src_path) + .status() + .expect("spawn rustc to build the fake helper"); + assert!(status.success(), "rustc failed to compile the fake helper"); + exe_path + } + + #[test] + fn batch_parse_persists_helper_warnings_alongside_the_cached_refs() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let key = "csharp Ns . V#Validate()."; + + // Batch-find-refs output exactly as the helper writes it: version + // 1.0, one resolved symbol, a `warnings` array reporting what it + // survived. + let output_path = dir.path().join("batch-refs.json"); + let doc = serde_json::json!({ + "version": scip_parse::SUPPORTED_INDEX_VERSION, + "results": [{ + "symbol": key, + "references": [ + {"file": "src/generated.cs", "start_line": 11, "end_line": 11, "kind": "reference"}, + ], + "warnings": [ + "could not compile project 'Broken' — its symbols are missing from the map", + ], + }], + }); + std::fs::write(&output_path, doc.to_string()).unwrap(); + + let indexer = CSharpSymbolIndexer::new(); + let cached = indexer + .parse_and_cache_batch_refs(&db, &output_path) + .unwrap(); + assert_eq!(cached, 1, "the one result must be cached"); + + // The refs landed in the cache... + let refs = indexer.find_references_for_key(&db, key).unwrap(); + assert!( + refs.iter() + .any(|r| r.file == Path::new("src/generated.cs") && r.kind == "reference"), + "the batch refs must be cached, got {refs:?}" + ); + // ...and the warning the helper reported must be persisted WITH + // them — the cached-partial fact is one atomic fact. + let warnings = indexer.lookup_warnings(&db, key); + assert_eq!(warnings.len(), 1, "the batch warning must be persisted"); + assert!( + warnings[0].contains("could not compile project 'Broken'"), + "warning text must round-trip, got: {}", + warnings[0] + ); + + // A later batch run WITHOUT the warnings field (an old helper) must + // clear the stale warning: absence = complete, so a clean + // re-resolution stops claiming partiality. + let clean_path = dir.path().join("batch-refs-clean.json"); + let clean = serde_json::json!({ + "version": scip_parse::SUPPORTED_INDEX_VERSION, + "results": [{ + "symbol": key, + "references": [], + }], + }); + std::fs::write(&clean_path, clean.to_string()).unwrap(); + indexer + .parse_and_cache_batch_refs(&db, &clean_path) + .unwrap(); + assert!( + indexer.lookup_warnings(&db, key).is_empty(), + "a warnings-free batch result must clear the stale warning" + ); + } + + #[test] + #[serial_test::serial] + fn lazy_cache_miss_persists_helper_warnings_alongside_the_cached_refs() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + // The .sln sits at db_path.parent() — the repo_path the lazy path + // falls back to when scip_meta carries no META_REPO_PATH. + std::fs::write(dir.path().join("FakeSolution.sln"), "").unwrap(); + + // Cache-miss fixture: one DEFINITION-only entry (no cached refs, no + // legacy reference kinds), so the lazy find-refs path runs. + let key = "csharp Ns . V#Validate()."; + { + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let symbols: Database = env + .open_database(&wtxn, Some(SCIP_DB_NAME)) + .unwrap() + .unwrap(); + symbols + .put( + &mut wtxn, + key, + &serialize_refs(&[StoredReference { + file: PathBuf::from("src/v.cs"), + start_line: 1, + end_line: 1, + kind: "definition".into(), + }]) + .unwrap(), + ) + .unwrap(); + wtxn.commit().unwrap(); + } + + let helper_dir = dir.path().join("helper"); + std::fs::create_dir(&helper_dir).unwrap(); + let helper = build_fake_csharp_helper(&helper_dir); + let _guard = + crate::testing::EnvRestore::set(&[(HELPER_ENV_VAR, helper.to_string_lossy().as_ref())]); + + let indexer = CSharpSymbolIndexer::new(); + let refs = indexer.find_references_for_key(&db, key).unwrap(); + + // The helper's reference made it through the whole pipe: resident + // handshake fail → one-shot spawn → JSON parse → return. + assert!( + refs.iter() + .any(|r| r.file == Path::new("src/generated.cs") && r.kind == "reference"), + "the helper's reference must be returned, got {refs:?}" + ); + // THE PRODUCER ASSERTION: the helper's warning was persisted by the + // SAME write phase that cached the refs (find_refs_for_canonical_key). + let warnings = indexer.lookup_warnings(&db, key); + assert_eq!( + warnings.len(), + 1, + "the lazy path must persist the helper's warning together with the cache" + ); + assert!( + warnings[0].contains("FindReferencesAsync failed"), + "warning text must round-trip, got: {}", + warnings[0] + ); + } } diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs new file mode 100644 index 00000000..31991aa0 --- /dev/null +++ b/src/symbols/csharp_tests.rs @@ -0,0 +1,126 @@ +//! Helper stderr routing tests (`csharp.rs`). Sibling `_tests.rs` file +//! per repo convention. + +use super::csharp::{drain_pipe_to_tracing, is_helper_warning_line}; +use std::io::Cursor; +use std::sync::Mutex; + +fn drained(input: &[u8]) -> Vec { + let out: Mutex> = Mutex::new(Vec::new()); + drain_pipe_to_tracing(Cursor::new(input.to_vec()), |line| { + out.lock().expect("test mutex").push(line.to_string()); + }); + out.into_inner().expect("test mutex") +} + +#[test] +fn drain_survives_bad_bytes_and_strips_eol() { + let mut raw = b"first line\n".to_vec(); + raw.extend_from_slice(b"bad \xFF\xFE bytes\n"); // invalid UTF-8 mid-stream + raw.extend_from_slice(b"crlf line\r\n"); + raw.extend_from_slice(b"no trailing newline"); + + let lines = drained(&raw); + + // The defect this pins: lines().map_while(Result::ok) ended the drain + // permanently at the non-UTF-8 line, silently dropping lines 3-4 and + // eventually stalling the pipe. All four lines must come through. + assert_eq!(lines.len(), 4, "drain stopped early: {lines:?}"); + assert_eq!(lines[0], "first line"); + assert!( + lines[1].starts_with("bad ") && lines[1].ends_with(" bytes"), + "expected lossy-decoded line, got: {:?}", + lines[1] + ); + assert_eq!(lines[2], "crlf line"); + assert_eq!(lines[3], "no trailing newline"); +} + +#[test] +fn drain_stops_at_eof_and_handles_empty_pipe() { + assert!(drained(b"").is_empty()); + assert_eq!(drained(b"only\n"), vec!["only".to_string()]); +} + +/// Read impl whose first read fails with `Interrupted` (EINTR), then +/// behaves like a normal pipe. +struct InterruptedOnce { + armed: bool, + inner: Cursor>, +} + +impl std::io::Read for InterruptedOnce { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.armed { + self.armed = false; + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "eintr", + )); + } + self.inner.read(buf) + } +} + +#[test] +fn drain_retries_after_transient_interrupted_read() { + // Contract: a transient Interrupted read must not end the drain. std's + // own read_until already retries Interrupted internally (rustc 1.98, + // library/std/src/io/mod.rs), so the local arm in drain_pipe_to_tracing + // is belt-and-braces and unreachable via its internal BufReader — this + // test pins the observable contract, not that arm. + let mut out: Vec = Vec::new(); + drain_pipe_to_tracing( + InterruptedOnce { + armed: true, + inner: Cursor::new(b"before\nafter\n".to_vec()), + }, + |line| out.push(line.to_string()), + ); + assert_eq!(out, vec!["before".to_string(), "after".to_string()]); +} + +#[test] +fn helper_warning_classification_table() { + let cases: [(&str, bool); 10] = [ + // Helper's own workspace-failure warnings (WorkspaceFailed handler). + ( + "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\r\\p.csproj' with message: Dependency specified was X but ended up with X 1.2.3", + true, + ), + ( + "[WARN] Solution load partially failed (InvalidOperationException: boom); continuing with 13 loaded project(s).", + true, + ), + ( + "serve: [WARN] no projects loaded — cannot serve.", + true, + ), + // Bare MSBuildWorkspace diagnostic without the helper prefix. + ( + "[Failure] Msbuild failed when processing the file 'C:\\r\\p.csproj'", + true, + ), + ("[WARN] Project file not found: C:\\r\\missing.csproj", true), + // Progress / info lines must NOT escalate to warn. + ( + "[INFO] Skipping unsupported project type: C:\\r\\x.shproj", + false, + ), + ("Loading solution: C:\\r\\x.sln", false), + ("Loaded 13 project(s) from filtered solution", false), + ( + "MSBuild: registering '.NET SDK' v10.0.303 at C:\\Program Files\\dotnet\\sdk\\10.0.303", + false, + ), + ("Index written to: C:\\tmp\\out.json", false), + ]; + + for (line, expected) in cases { + assert_eq!( + is_helper_warning_line(line), + expected, + "misclassified line: {line}" + ); + } +} diff --git a/src/symbols/lock_visibility_tests.rs b/src/symbols/lock_visibility_tests.rs new file mode 100644 index 00000000..70106096 --- /dev/null +++ b/src/symbols/lock_visibility_tests.rs @@ -0,0 +1,101 @@ +//! Lock-visibility verification for todo #96 step 5. +//! +//! The busy answer deliberately carries no `lock_status` field — that is +//! only warranted if readers can actually BLOCK on an index the indexer is +//! writing. LMDB is MVCC: a read transaction never waits on the single +//! writer. These tests pin that property at the exact level `find_impact` +//! reads (a `read_txn` on the shared env): a reader issued while another +//! thread holds an uncommitted write transaction must complete promptly. +//! If LMDB semantics ever changed such that readers blocked, the 5s +//! timeout here fails — and `lock_status` in the busy envelope becomes the +//! follow-up, not before. + +use std::time::Duration; + +use heed::types::Str; +use heed::Database; + +#[test] +fn readers_do_not_block_while_a_write_txn_holds_the_write_lock() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(10 * 1024 * 1024).max_dbs(3); + // SAFETY: same flags as every env in this repo (see BASE_ENV_FLAGS). + unsafe { + opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS); + } + let env = unsafe { opts.open(&dir) }.expect("open env"); + + let db: Database = { + let mut wtxn = env.write_txn().expect("setup wtxn"); + let db = env.create_database(&mut wtxn, Some("t")).expect("db"); + wtxn.commit().expect("setup commit"); + db + }; + + // Hold the write lock with an UNCOMMITTED write. + let mut wtxn = env.write_txn().expect("wtxn"); + db.put(&mut wtxn, "k", "v1").expect("put"); + + // The find_impact shape: a reader on the same env, another thread. + let reader_env = env.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let rtxn = reader_env.read_txn(); + let outcome = rtxn.and_then(|rt| db.get(&rt, "k").map(|v| v.map(str::to_string))); + let _ = tx.send(outcome); + }); + + let read_while_locked = rx + .recv_timeout(Duration::from_secs(5)) + .expect("reader must FINISH while the write lock is held — it blocked!"); + assert!( + matches!(read_while_locked, Ok(None)), + "uncommitted write must be invisible: {read_while_locked:?}" + ); + + // Control: the write was real — after commit it is visible. + wtxn.commit().expect("commit"); + let rtxn = env.read_txn().expect("rtxn"); + assert_eq!(db.get(&rtxn, "k").expect("get"), Some("v1")); +} + +#[test] +fn concurrent_reader_and_writer_do_not_deadlock() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(10 * 1024 * 1024).max_dbs(3); + // SAFETY: same flags as every env in this repo (see BASE_ENV_FLAGS). + unsafe { + opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS); + } + let env = unsafe { opts.open(&dir) }.expect("open env"); + + let db: Database = { + let mut wtxn = env.write_txn().expect("setup wtxn"); + let db = env.create_database(&mut wtxn, Some("t")).expect("db"); + wtxn.commit().expect("setup commit"); + db + }; + + // Writer and reader interleaved on two threads — the steady state of a + // serve with an active indexer and a busy find_impact. + let writer_env = env.clone(); + let writer = std::thread::spawn(move || { + for i in 0..25u32 { + let mut wtxn = writer_env.write_txn().expect("wtxn"); + db.put(&mut wtxn, "k", i.to_string().as_str()).expect("put"); + wtxn.commit().expect("commit"); + } + }); + let reader_env = env.clone(); + let reader = std::thread::spawn(move || { + for _ in 0..25u32 { + let rtxn = reader_env.read_txn().expect("rtxn"); + let _ = db.get(&rtxn, "k").expect("get"); + drop(rtxn); + } + }); + writer.join().expect("writer"); + reader.join().expect("reader"); +} diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index a20f6d1d..9295618d 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -8,15 +8,33 @@ //! `SymbolIndexer` impls here. pub mod csharp; +pub mod resident; pub mod scip_parse; pub mod scip_proto; pub mod typescript; use std::path::{Path, PathBuf}; -use anyhow::Result; +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; +// ── Helper rendering ────────────────────────────────────────────── + +/// Platform-stable rendering of a helper process's exit status. +/// +/// `ExitStatus`'s `Display` prints `exit code: N` on Windows but +/// `exit status: N` on Unix — text that is persisted into index meta +/// tables and asserted by tests must not depend on the host OS (the +/// Linux CI red after #238 was exactly this drift). Signal-terminated +/// processes (Unix `code() == None`) have no portable code and fall +/// back to the platform's own rendering. +pub(crate) fn exit_status_text(status: &std::process::ExitStatus) -> String { + match status.code() { + Some(code) => format!("exit code: {code}"), + None => status.to_string(), + } +} + // ── Common types ────────────────────────────────────────────────── /// A resolved reference to a symbol — file, line range, and kind. @@ -35,16 +53,114 @@ pub struct SymbolReference { /// Result of a `find_impact` query. #[derive(Debug, Clone, Serialize)] pub struct FindImpactResult { - /// Canonical SCIP symbol string, e.g. `csharp . . . FieldDefinition#Validate().` + /// The query as asked — a symbol name, `file:line`, or explicit key. pub symbol: String, + /// The canonical SCIP key actually selected, e.g. + /// `csharp . . . FieldDefinition#Validate().` Present only when the + /// query resolved to exactly one stored symbol; absent on an ambiguous + /// answer (a separate envelope) or when nothing matched. The selector + /// the issue-specified contract turns on: echo is `symbol`, identity + /// is this field. + #[serde(skip_serializing_if = "Option::is_none")] + pub resolved_symbol: Option, /// Resolved references. pub references: Vec, + /// Completeness warnings for this answer. Non-empty means the + /// reference list may be INCOMPLETE — a helper failure was survived + /// rather than fatal (a project that would not compile, an exception + /// during reference resolution, a non-zero scip-typescript exit) and + /// the partial result was kept. Entries name what failed. Omitted when + /// empty: absent means the answer is as complete as the index knows. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub warnings: Vec, /// Seconds since the symbol index was last rebuilt. pub index_age_seconds: u64, /// Language that produced this result. pub language: String, /// Scope that was searched, e.g. `"project:example-org"`. pub scope: String, + /// Git HEAD sha the symbol index was built for, when recorded. Compare + /// with `current_head_sha` to spot index drift after a branch switch. + /// Drift is surfaced, never auto-reindexed (deliberate — see + /// `SymbolIndexer::index_head_sha`). Omitted when unknown. + #[serde(skip_serializing_if = "Option::is_none")] + pub index_head_sha: Option, + /// Repository HEAD at response time, when readable. Omitted when the + /// read fails (git unavailable, transient Windows handle race — must + /// never fail the response). + #[serde(skip_serializing_if = "Option::is_none")] + pub current_head_sha: Option, +} + +/// A `find_impact` query in the form the adapters resolve it. +/// +/// The three input variants of `FindImpactRequest` map onto this one-to-one; +/// `ExactKey` is the explicit selection a caller makes after an ambiguous +/// answer (or with any full key it already has). +#[derive(Debug, Clone)] +pub enum ImpactQuery { + /// Look up this exact canonical SCIP key. No fuzzy fallback: a key + /// that is not in the index is a `KeyMatch::NotFound`, never a guess. + ExactKey(String), + /// Resolve a possibly-fuzzy symbol name (e.g. `Validate`, + /// `FieldDefinition.Validate`) to one canonical key. + Name(String), + /// Resolve the symbol defined at a file position (1-based line). + Position { file: PathBuf, line: u32 }, +} + +/// Outcome of resolving an [`ImpactQuery`] against the index. +/// +/// Resolution is a plain LMDB read — fast, never invoking the SCIP +/// helper. Ambiguity is surfaced instead of silently picking the +/// shortest candidate (the pre-fix behaviour hid overloads this way). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum KeyMatch { + /// Exactly one stored symbol matches; this is the selection. + Resolved(String), + /// Several stored symbols match and the query alone cannot choose. + /// Candidates are sorted and deduplicated — re-query with + /// `ImpactQuery::ExactKey` for one of them. + Ambiguous(Vec), + /// Nothing in the index matches. + NotFound, +} + +/// Typed answer for an ambiguous `find_impact` query, serialized as JSON +/// in the tool-result text (same convention as [`SymbolLookupBusy`]). +/// +/// The MCP client re-calls with `symbol_key` set to one of `candidates`; +/// the server never silently picks on the client's behalf. +#[derive(Debug, Clone, Serialize)] +pub struct SymbolAmbiguity { + /// Always `true` — machine-branchable, like `busy`. + pub ambiguous: bool, + /// The query as asked. + pub query: String, + /// Every canonical key that matches, sorted. These are exact + /// `symbol_key` values, not prose. + pub candidates: Vec, + /// Actionable hint for the agent. + pub hint_for_agent: String, +} + +/// Resolve the repository HEAD sha for `repo_root` (40-hex), or `None` +/// when git is unavailable, the path is not a work tree, or the read +/// fails. Non-fatal BY DESIGN: a transient Windows git failure (see the +/// `GitHeadWatcher` note in `src/watch/mod.rs`) must never take down a +/// response that only wanted the fingerprint. +pub(crate) fn current_git_head(repo_root: &Path) -> Option { + let output = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(repo_root) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let sha = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let is_40_hex = sha.len() == 40 && sha.chars().all(|c| c.is_ascii_hexdigit()); + is_40_hex.then_some(sha) } /// Error returned when the symbol index is unavailable. @@ -58,6 +174,101 @@ pub struct SymbolIndexError { pub hint_for_agent: String, } +/// Structured busy answer returned when a `find_impact` lookup exceeds its +/// internal wall-clock budget. +/// +/// The MCP client must never be the timeout mechanism: when the budget +/// overruns, the server answers with this envelope (serialized as JSON in +/// the tool-result text) while the lookup keeps running in the background. +/// A caller can branch on `busy == true` instead of parsing an opaque +/// client-side timeout, and the retry hinted in `advice` is served from the +/// reference cache the running lookup will have populated. +#[derive(Debug, Clone, Serialize)] +pub struct SymbolLookupBusy { + /// Always `true`; makes the envelope self-describing. + pub busy: bool, + /// What is still running, e.g. `"resolving 'Ns.I.M' via the csharp SCIP helper"`. + pub state: String, + /// Wall-clock time the request waited before the budget overran. On the + /// retry answer for a tracked lookup this is the background lookup's + /// cumulative elapsed time instead. + pub waited_ms: u64, + /// Actionable retry hint, e.g. `"retry the same call in ~60s"`. + pub advice: String, + /// Machine-branchable retry interval in seconds, mirroring the prose in + /// `advice`: how long a harness with auto-retry semantics should sleep + /// before repeating the SAME call. Busy is progress, not failure. + pub retry_after_seconds: u64, +} + +/// Machine-branchable class of a failed `find_impact` lookup. +/// +/// `busy` needs no class here: an overran lookup answers with the typed +/// `SymbolLookupFailure`-free `SymbolLookupBusy` envelope instead. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SymbolLookupFailureClass { + /// The helper ran and failed (non-zero exit, crash, unusable output) + /// against a readable index. Retrying cannot succeed until the cause + /// changes; fall back to text search. + Failed, + /// No readable symbol index (`index_age` reports unknown): retrying + /// the lookup cannot succeed until the index is built or rebuilt. + Stale, +} + +/// Typed failure envelope for `find_impact` lookups, serialized as JSON in +/// the tool-result text (same convention as `SymbolIndexError`). Replaces +/// the former soft-string render (`Symbol lookup failed: ...`) so an agent +/// can branch on `class` instead of parsing prose. +#[derive(Debug, Clone, Serialize)] +pub struct SymbolLookupFailure { + /// Rendered `{:#}` error chain. + pub error: String, + /// Machine-branchable failure class. + pub class: SymbolLookupFailureClass, + /// Actionable hint for the agent. + pub hint_for_agent: String, +} + +impl SymbolLookupFailure { + /// A failed lookup against a readable index. + pub fn failed(error_chain: impl Into) -> Self { + Self { + error: error_chain.into(), + class: SymbolLookupFailureClass::Failed, + hint_for_agent: "The SCIP helper failed for this lookup. Do not retry the same call \ + immediately; fall back to `find` with kind=\"usages\" (text-based) \ + and/or reindex the project to rebuild the symbol index." + .to_string(), + } + } + + /// A failed lookup with an unreadable/absent symbol index. + pub fn stale(error_chain: impl Into) -> Self { + Self { + error: error_chain.into(), + class: SymbolLookupFailureClass::Stale, + hint_for_agent: "No readable symbol index for this project. Build or rebuild it \ + first (`codesearch index` / `index reindex`), then retry the \ + same call." + .to_string(), + } + } + + /// Classify a lookup failure by the index age reported for the same db: + /// an unknown age (`u64::MAX`, what `index_age` returns whenever the + /// index cannot be opened or read) means stale; anything else means the + /// index was readable and the lookup itself failed. + pub fn classify(error_chain: impl Into, index_age_seconds: u64) -> Self { + if index_age_seconds == u64::MAX { + Self::stale(error_chain) + } else { + Self::failed(error_chain) + } + } +} + /// Which files/projects to reindex. #[derive(Debug, Clone)] #[allow(dead_code)] @@ -79,6 +290,76 @@ pub enum RebuildScope { }, } +// ── Shared SCIP environment ────────────────────────────────────── + +/// Open (or reuse) the process-wide shared SCIP LMDB environment for `db_path`. +/// +/// Both the C# and TypeScript adapters store symbol data in the same +/// `db_path/scip` directory, and LMDB allows exactly ONE open environment per +/// directory per process. Historically every operation opened its own +/// short-lived env, so two overlapping operations — e.g. a watcher-triggered +/// rebuild starting while a lazy `find-refs` call held its env for minutes — +/// tripped the double-open guard and one side failed outright +/// (`LMDB double-open prevented`, surfaced as a red `C#!` in the TUI). +/// Routing every open through this getter hands all concurrent users the SAME +/// environment: writers serialise on LMDB's single-writer mutex, readers never +/// block, and the double-open error class cannot occur. +pub(crate) fn get_shared_scip_env( + db_path: &Path, +) -> Result> { + let scip_dir = db_path.join("scip"); + std::fs::create_dir_all(&scip_dir) + .with_context(|| format!("Failed to create SCIP directory: {}", scip_dir.display()))?; + + crate::lmdb_registry::get_or_open_shared_env( + &scip_dir, + &format!("SCIP({})", db_path.display()), + |opts| { + // map_size is virtual address space (not RSS); the OS only faults + // in written pages. Read once per env lifetime. + let map_size_mb = std::env::var(crate::constants::SCIP_LMDB_MAP_SIZE_MB_ENV) + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(crate::constants::SCIP_LMDB_DEFAULT_MAP_SIZE_MB); + opts.map_size(map_size_mb * 1024 * 1024).max_dbs(10); + // SAFETY: `NO_TLS` only changes reader-slot tracking. See `BASE_ENV_FLAGS`. + unsafe { opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS) }; + }, + |env| { + // Pre-create every named database (both languages') exactly once + // per env session: LMDB requires named DBs to exist before they + // can be opened in read txns. + let mut wtxn = env.write_txn()?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_SYMBOLS_DB_NAME), + )?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_META_DB_NAME), + )?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_POSITION_DB_NAME), + )?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_SIMPLE_NAMES_DB_NAME), + )?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_REF_CACHE_DB_NAME), + )?; + env.create_database::( + &mut wtxn, + Some(crate::constants::SCIP_REF_WARNINGS_DB_NAME), + )?; + wtxn.commit()?; + Ok(()) + }, + ) +} + /// Summary returned after a rebuild completes. #[derive(Debug, Clone)] #[allow(dead_code)] @@ -126,21 +407,51 @@ pub trait SymbolIndexer: Send + Sync { scope: RebuildScope, ) -> Result; - /// Return the symbol's references from the LMDB store. - fn find_references(&self, db_path: &Path, symbol: &str) -> Result>; + /// Resolve a `find_impact` query to canonical SCIP key(s), exposing + /// ambiguity instead of silently choosing among candidates. + /// + /// Plain LMDB reads only — never invokes the SCIP helper, so callers + /// can (and the find_impact handler does) run this BEFORE the + /// budget-tracked reference fetch: an ambiguous answer is instant. + /// `KeyMatch::Resolved` is the selected canonical identity; the + /// historical behaviour picked the shortest fuzzy match silently, + /// which hid overloads from the caller. + fn resolve_query(&self, db_path: &Path, query: &ImpactQuery) -> Result; - /// Look up references by file-position instead of symbol name. - /// Resolves the position to a canonical SCIP symbol first. - fn find_references_by_position( + /// Return the symbol's references for one EXACT canonical key. + /// + /// This is the expensive half of a lookup: on the C# adapter a cold + /// reference cache triggers the `scip-csharp find-refs` subprocess + /// (minutes on a large solution), which is why the find_impact + /// handler budgets and tracks THIS call and not `resolve_query`. + fn find_references_for_key( &self, db_path: &Path, - file: &Path, - line: u32, + canonical_key: &str, ) -> Result>; + /// Completeness warnings for one canonical key's stored answer — a + /// plain LMDB read, NEVER invoking a helper. Non-empty means the + /// references `find_references_for_key` returns for this key may be + /// INCOMPLETE: a partial helper run was survived and cached, and the + /// warnings were persisted with it so the partial answer can never + /// pass for complete. Entries name what failed. + fn lookup_warnings(&self, db_path: &Path, canonical: &str) -> Vec; + /// How old is the current symbol index (seconds since last rebuild)? fn index_age(&self, db_path: &Path) -> u64; + /// The git HEAD sha the current symbol index was built for, when + /// recorded. `None` means unknown (pre-fingerprint index, or git was + /// unreadable at build time). Compare with the repository's current + /// HEAD to make index drift after a branch switch visible — drift is + /// surfaced, never auto-reindexed (deliberate: reindexing a large + /// solution on every branch switch would thrash; refresh stays an + /// explicit operator action). + fn index_head_sha(&self, _db_path: &Path) -> Option { + None + } + /// Whether the helper binary for this language is available. fn is_available(&self) -> bool; @@ -235,3 +546,41 @@ impl Default for SymbolIndexerRegistry { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The scip getter must hand out ONE shared env per `db_path` — the exact + /// property that stops a rebuild and an in-flight lazy find-refs (or the + /// TypeScript adapter) from failing each other with the double-open error. + #[test] + fn shared_scip_env_is_shared_across_calls() { + let dir = tempfile::TempDir::new().unwrap(); + let db_path = dir.path().join("codesearch.db"); + + let env1 = get_shared_scip_env(&db_path).unwrap(); + let env2 = get_shared_scip_env(&db_path).unwrap(); + assert!(std::sync::Arc::ptr_eq(&env1, &env2)); + } +} + +/// Lock-visibility verification (todo #96 step 5): readers never block on +/// an open write transaction — the property that lets `find_impact` serve +/// during an indexer rebuild without needing a `lock_status` in the busy +/// envelope. Sibling `_tests.rs` file per repo convention. +#[cfg(test)] +#[path = "lock_visibility_tests.rs"] +mod lock_visibility_tests; + +/// Resident-helper WorkspacePool tests (todo #115). Sibling `_tests.rs` +/// file per repo convention. +#[cfg(test)] +#[path = "resident_tests.rs"] +mod resident_tests; + +/// Helper stderr routing tests (`csharp.rs`). Sibling `_tests.rs` file +/// per repo convention. +#[cfg(test)] +#[path = "csharp_tests.rs"] +mod csharp_tests; diff --git a/src/symbols/resident.rs b/src/symbols/resident.rs new file mode 100644 index 00000000..6548990b --- /dev/null +++ b/src/symbols/resident.rs @@ -0,0 +1,503 @@ +//! Resident SCIP helper (serve mode) + `WorkspacePool` — todo #115. +//! +//! The `scip-csharp serve` subcommand loads a solution's Roslyn workspace +//! ONCE and then answers find-refs requests as JSON lines over stdin/stdout. +//! The pool below owns helper lifecycle: admission (max N resident, LRU +//! eviction), per-workspace heap caps, and idle teardown. Eviction is safe — +//! resolved references persist in the LMDB ref cache, so only latency is +//! lost, never data. +//! +//! Memory model (the governor): `MAX_RESIDENT × heap cap` bounds total +//! helper memory. A runaway workspace fails fast at its cap and the typed +//! `failed` path turns that into an agent-actionable answer. + +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::SymbolReference; + +/// References plus the completeness warnings from one resident find-refs +/// call. The warnings must travel WITH the references, not alongside them +/// in a log line: a partial answer that gets cached must stay partial +/// forever after unless the warnings are persisted with it. +#[derive(Debug, Default)] +pub(crate) struct ResidentRefs { + pub references: Vec, + /// Non-empty means `references` may be incomplete (a project failed to + /// compile, FindReferencesAsync threw). Empty = complete. + pub warnings: Vec, +} + +/// Behaviour seam so the pool can be unit-tested without real processes. +pub(crate) trait ClientLike: Send + Sync { + fn find_refs(&self, symbol: &str) -> Result; + /// Kill the helper process. Must be idempotent. + fn kill(&self); +} + +type SpawnFn = Box Result> + Send + Sync>; + +/// The in-flight reservation taken under the pool lock and released by the +/// caller-owned parts — eviction may remove the map entry while the request +/// runs, so release must not depend on the map. +type Reservation = (Arc, Arc, Arc); + +/// A live `scip-csharp serve` child: spawned with a heap cap, handshaken on +/// the ready line, then strictly request-response. Kill-on-drop is the +/// backstop; the pool kills explicitly on eviction. +pub(crate) struct ServeClient { + /// Mutexes over the two pipe halves; request-response is strictly + /// sequential so no condition variable is needed. + stdin: Mutex, + stdout: Mutex>, + child: Mutex, +} + +impl ServeClient { + /// Spawn `scip-csharp serve --solution ` and wait for the ready + /// handshake. The workspace load takes MINUTES on large solutions — this + /// call blocks until the helper reports ready, which is why callers run + /// it on the blocking pool and why the find_impact budget exists. + pub(crate) fn spawn(helper: &Path, solution: &Path, heap_cap_bytes: u64) -> Result { + // DOTNET_GCHeapHardLimit is interpreted by the .NET runtime as hex. + let heap_cap_hex = format!("{heap_cap_bytes:X}"); + let mut child = Command::new(helper) + .arg("serve") + .arg("--solution") + .arg(solution) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + // Helper stderr is diagnostics (load progress + workspace-load + // warnings). Pipe it and drain via tracing: an inherited stderr + // bypasses the file-only serve logger entirely and sprays raw + // MSBuild output straight onto the TUI, scrambling it. + .stderr(Stdio::piped()) + .env("DOTNET_GCHeapHardLimit", heap_cap_hex) + .spawn() + .with_context(|| { + format!( + "Failed to spawn resident scip-csharp serve at {}", + helper.display() + ) + })?; + + let stdin = child + .stdin + .take() + .context("serve helper has no stdin pipe")?; + let stdout = child + .stdout + .take() + .context("serve helper has no stdout pipe")?; + + // Drain helper stderr through tracing (warnings classified as warn). + // Detached: the thread lives until the helper dies (EOF), so kill and + // Drop teardown need no coordination with it. Without a concurrent + // drain the pipe buffer would fill during the minutes-long workspace + // load and block the helper. + if let Some(stderr) = child.stderr.take() { + let label = solution + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| solution.display().to_string()); + thread::spawn(move || { + super::csharp::drain_pipe_to_tracing(stderr, |line| { + if !line.is_empty() { + super::csharp::emit_helper_stderr_line("scip-csharp serve", &label, line); + } + }); + }); + } + + let client = Self { + stdin: Mutex::new(stdin), + stdout: Mutex::new(BufReader::new(stdout)), + child: Mutex::new(child), + }; + + client + .wait_ready() + .context("serve helper failed to become ready")?; + Ok(client) + } + + /// Blocks until the ready handshake line arrives. No timeout: a large + /// solution loads for minutes BY DESIGN, and the find_impact budget (not + /// a watchdog here) is what turns that wait into a structured busy + /// answer for the caller while this lookup continues detached. + fn wait_ready(&self) -> Result<()> { + let line = self.read_response_line()?; + let response: ServeResponse = serde_json::from_str(&line) + .with_context(|| format!("serve handshake is not valid JSON: {line}"))?; + if !response.ok || response.ready != Some(true) { + anyhow::bail!( + "serve handshake failed: {}", + response.error.unwrap_or_default() + ); + } + Ok(()) + } + + /// Reads one protocol line from stdout. An empty read (EOF) means the + /// helper died — surface as an error so the caller's fallback kicks in. + fn read_response_line(&self) -> Result { + let mut line = String::new(); + let n = { + let mut reader = self.stdout.lock().expect("serve stdout mutex poisoned"); + reader.read_line(&mut line)? + }; + if n == 0 { + anyhow::bail!("serve helper closed stdout (process died?)"); + } + Ok(line) + } +} + +// ── Protocol wire types (snake_case — matches the helper's serializer) ── + +#[derive(serde::Deserialize)] +struct ServeResponse { + ok: bool, + #[serde(default)] + error: Option, + #[serde(default)] + ready: Option, + #[serde(default)] + result: Option, +} + +#[derive(serde::Deserialize)] +struct ServeResult { + #[serde(default)] + references: Vec, + /// Absent in helper output from before warnings existed — default to + /// empty so old binaries keep parsing as "complete". + #[serde(default)] + warnings: Vec, +} + +#[derive(serde::Deserialize)] +struct ServeRef { + file: String, + start_line: u32, + end_line: u32, + #[serde(default = "default_ref_kind")] + kind: String, +} + +fn default_ref_kind() -> String { + "reference".to_string() +} + +impl ClientLike for ServeClient { + fn find_refs(&self, symbol: &str) -> Result { + // Sequential protocol: write request, then read exactly one response. + { + let mut stdin = self.stdin.lock().expect("serve stdin mutex poisoned"); + let request = serde_json::json!({ "op": "find-refs", "symbol": symbol }); + writeln!(stdin, "{request}").context("failed to write find-refs request")?; + stdin.flush().context("failed to flush find-refs request")?; + } + + let line = self.read_response_line()?; + let response: ServeResponse = serde_json::from_str(&line) + .with_context(|| format!("serve response is not valid JSON: {line}"))?; + + if !response.ok { + anyhow::bail!( + "serve find-refs failed: {}", + response + .error + .unwrap_or_else(|| "unknown error".to_string()) + ); + } + + let result = response + .result + .context("serve find-refs response has no result")?; + Ok(ResidentRefs { + references: result + .references + .into_iter() + .map(|r| SymbolReference { + file: PathBuf::from(r.file), + start_line: r.start_line, + end_line: r.end_line, + kind: r.kind, + }) + .collect(), + warnings: result.warnings, + }) + } + + fn kill(&self) { + if let Ok(mut child) = self.child.lock() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +impl Drop for ServeClient { + fn drop(&mut self) { + // Backstop: if the pool ever loses the last reference without an + // explicit kill, the child must not outlive the host. + if let Ok(mut child) = self.child.lock() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +// ── WorkspacePool — admission control IS the memory governor ──────────── + +/// Per-repo state inside the pool. The Arcs are cloned into every in-flight +/// request so eviction can observe "still in use" and defer the kill — the +/// counter-then-teardown discipline: in_flight is incremented under the same +/// pool lock that decides eviction, so an eviction decision can never miss an +/// in-flight request that grabbed its handle first. +struct PoolEntry { + client: Arc, + in_flight: Arc, + doomed: Arc, + last_used: Instant, +} + +pub(crate) struct WorkspacePool { + /// Keyed by solution path (the workspace identity). + entries: Mutex>, + /// Per-key spawn mutexes: two concurrent lookups on the SAME repo must + /// not both pay the minutes-long workspace load. Different repos spawn + /// in parallel. Entries are never removed (bounded by repo count). + spawn_locks: Mutex>>>, + max_resident: usize, + idle: Duration, + heap_cap: u64, + spawn_fn: SpawnFn, +} + +impl WorkspacePool { + #[allow(dead_code)] + pub(crate) fn new( + max_resident: usize, + idle: Duration, + heap_cap: u64, + spawn_fn: SpawnFn, + ) -> Self { + Self { + entries: Mutex::new(HashMap::new()), + spawn_locks: Mutex::new(HashMap::new()), + max_resident, + idle, + heap_cap, + spawn_fn, + } + } + + /// Resolve references for `symbol` via the resident workspace for + /// `solution`, spawning the helper if the repo has no resident workspace + /// yet. Errors here are expected and handled by the caller's one-shot + /// fallback (spawn failure, heap-cap death, eviction race, protocol). + /// Completeness warnings travel with the references — the caller + /// persists them alongside the cached refs. + pub(crate) fn find_refs( + &self, + helper: &Path, + solution: &Path, + symbol: &str, + ) -> Result { + // Single-flight per repo: the second concurrent lookup on the same + // repo waits for the first one's spawn instead of duplicating the + // minutes-long workspace load. + let key_lock = { + let mut locks = self.spawn_locks.lock().expect("spawn locks poisoned"); + locks + .entry(solution.to_path_buf()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + }; + let _key_guard = key_lock.lock().expect("per-key spawn mutex poisoned"); + + if let Some((client, in_flight, doomed)) = self.acquire_existing(solution) { + let outcome = client.find_refs(symbol); + self.release_parts(&in_flight, &doomed, &client); + return outcome; + } + + // Spawn OUTSIDE the pool lock — it takes minutes and must not block + // lookups on other repos (admission still happens under the lock). + self.enforce_admission(); + let client = (self.spawn_fn)(helper, solution, self.heap_cap)?; + let in_flight = Arc::new(AtomicUsize::new(1)); + let doomed = Arc::new(AtomicBool::new(false)); + + { + let mut entries = self.entries.lock().expect("workspace pool poisoned"); + // Race: another thread inserted this repo while we spawned. Kill + // OUR fresh workspace and use the existing resident one instead — + // two workspaces for one solution is exactly what the governor + // exists to prevent. The reservation is taken under the same + // lock that observes the existing entry. + if let Some(existing) = entries.get_mut(solution) { + existing.last_used = Instant::now(); + existing.in_flight.fetch_add(1, Ordering::SeqCst); + let (c, inf, d) = ( + existing.client.clone(), + existing.in_flight.clone(), + existing.doomed.clone(), + ); + drop(entries); + drop(client); // Drop impl kills our redundant child + let outcome = c.find_refs(symbol); + self.release_parts(&inf, &d, &c); + return outcome; + } + entries.insert( + solution.to_path_buf(), + PoolEntry { + client: client.clone(), + in_flight: Arc::clone(&in_flight), + doomed: Arc::clone(&doomed), + last_used: Instant::now(), + }, + ); + } + + let outcome = client.find_refs(symbol); + self.release_parts(&in_flight, &doomed, &client); + outcome + } + + /// Get the resident client for a repo, if any (TTL-checked). Takes the + /// in-flight reservation under the pool lock and returns the reservation + /// parts to the caller — release must NOT depend on the map, because the + /// entry may be evicted (and its map slot removed) while the request is + /// in flight. + fn acquire_existing(&self, solution: &Path) -> Option { + let mut to_kill: Vec> = Vec::new(); + let resident = { + let mut entries = self.entries.lock().expect("workspace pool poisoned"); + + // Lazy TTL reap while we hold the lock (counter-then-teardown: + // in-flight entries are never reaped here — they can't be, the + // filter requires in_flight == 0). + let expired: Vec = entries + .iter() + .filter(|(_, e)| { + e.last_used.elapsed() > self.idle && e.in_flight.load(Ordering::SeqCst) == 0 + }) + .map(|(k, _)| k.clone()) + .collect(); + for k in expired { + if let Some(e) = entries.remove(&k) { + to_kill.push(e.client); + } + } + + let resident = match entries.get_mut(solution) { + Some(entry) => { + entry.last_used = Instant::now(); + entry.in_flight.fetch_add(1, Ordering::SeqCst); + Some(( + entry.client.clone(), + entry.in_flight.clone(), + entry.doomed.clone(), + )) + } + None => None, + }; + resident + }; + for c in to_kill { + c.kill(); // OS call — outside the pool lock + } + resident + } + + /// Evict as many residents as needed to admit a new workspace — LRU + /// first. The admission decision AND the in_flight check happen under + /// the same lock (counter-then-teardown); kills happen outside it. + fn enforce_admission(&self) { + let mut victims: Vec> = Vec::new(); + { + let mut entries = self.entries.lock().expect("workspace pool poisoned"); + while entries.len() >= self.max_resident { + let Some(lru_key) = entries + .iter() + .min_by_key(|(_, e)| e.last_used) + .map(|(k, _)| k.clone()) + else { + break; + }; + if let Some(e) = entries.remove(&lru_key) { + if e.in_flight.load(Ordering::SeqCst) == 0 { + victims.push(e.client); + } else { + // In flight: defer the kill to its release — the + // request fails (the pipe dies with the process) and + // the caller's one-shot fallback keeps it correct. + e.doomed.store(true, Ordering::SeqCst); + } + } + } + } + for v in victims { + v.kill(); + } + } + + /// Release the in-flight reservation. Works from the caller-owned Arcs, + /// NOT the map: the entry may have been evicted (map slot removed) while + /// this request was in flight, and the deferred doomed-kill must still + /// fire exactly once — on the decrement that reaches zero. + fn release_parts( + &self, + in_flight: &Arc, + doomed: &Arc, + client: &Arc, + ) { + // Reached zero on OUR decrement while doomed → the deferred kill is + // ours to perform (eviction deferred it under the pool lock). + if in_flight.fetch_sub(1, Ordering::SeqCst) == 1 && doomed.load(Ordering::SeqCst) { + client.kill(); + } + } +} + +fn env_resolved(env: &str, default: T) -> T { + match std::env::var(env) { + Ok(raw) => raw.trim().parse().unwrap_or(default), + Err(_) => default, + } +} + +/// The process-global pool. Config comes from env at first use (same pattern +/// as the find_impact budget resolver). +pub(crate) static WORKSPACE_POOL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + let spawn: SpawnFn = Box::new(|helper: &Path, solution: &Path, heap_cap: u64| { + Ok(Arc::new(ServeClient::spawn(helper, solution, heap_cap)?)) + }); + WorkspacePool::new( + env_resolved( + crate::constants::SCIP_MAX_RESIDENT_WORKSPACES_ENV, + crate::constants::DEFAULT_SCIP_MAX_RESIDENT_WORKSPACES, + ), + Duration::from_secs(env_resolved( + crate::constants::SCIP_WORKSPACE_IDLE_SECS_ENV, + crate::constants::DEFAULT_SCIP_WORKSPACE_IDLE_SECS, + )), + env_resolved( + crate::constants::SCIP_WORKSPACE_HEAP_CAP_ENV, + crate::constants::DEFAULT_SCIP_WORKSPACE_HEAP_CAP, + ), + spawn, + ) + }); diff --git a/src/symbols/resident_tests.rs b/src/symbols/resident_tests.rs new file mode 100644 index 00000000..173fb751 --- /dev/null +++ b/src/symbols/resident_tests.rs @@ -0,0 +1,190 @@ +//! Tests for the resident-helper WorkspacePool (todo #115) — pure pool +//! logic against a mock client, no real processes involved. + +use super::resident::{ClientLike, ResidentRefs, WorkspacePool}; +use crate::symbols::SymbolReference; +use anyhow::Result; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +#[derive(Clone)] +struct MockState { + kill_count: Arc, + fail: Arc, + live_children: Arc, +} + +struct MockClient { + state: MockState, +} + +impl ClientLike for MockClient { + fn find_refs(&self, _symbol: &str) -> Result { + if self.state.fail.load(Ordering::SeqCst) { + anyhow::bail!("mock helper failed"); + } + Ok(ResidentRefs { + references: vec![SymbolReference { + file: PathBuf::from("src/Mock.cs"), + start_line: 1, + end_line: 1, + kind: "reference".to_string(), + }], + warnings: Vec::new(), + }) + } + + fn kill(&self) { + if self.state.kill_count.fetch_add(1, Ordering::SeqCst) == 0 { + self.state.live_children.fetch_sub(1, Ordering::SeqCst); + } + } +} + +struct Harness { + pool: WorkspacePool, + spawns: Arc, + state: MockState, +} + +fn harness(max: usize, idle: Duration) -> Harness { + let state = MockState { + kill_count: Arc::new(AtomicUsize::new(0)), + fail: Arc::new(AtomicBool::new(false)), + live_children: Arc::new(AtomicUsize::new(0)), + }; + let spawns = Arc::new(AtomicUsize::new(0)); + let spawn_state = state.clone(); + let spawn_spawns = spawns.clone(); + let spawn_fn = Box::new( + move |_helper: &Path, _solution: &Path, _cap: u64| -> Result> { + spawn_spawns.fetch_add(1, Ordering::SeqCst); + spawn_state.live_children.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(MockClient { + state: spawn_state.clone(), + })) + }, + ); + Harness { + pool: WorkspacePool::new(max, idle, 1, spawn_fn), + spawns, + state, + } +} + +fn sln(name: &str) -> PathBuf { + PathBuf::from(format!("C:\\code\\{name}\\src\\App.sln")) +} + +#[test] +fn resident_pool_admission_evicts_lru_when_full() { + let h = harness(2, Duration::from_secs(600)); + + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("b"), "Sym") + .unwrap(); + assert_eq!(h.spawns.load(Ordering::SeqCst), 2); + assert_eq!(h.state.kill_count.load(Ordering::SeqCst), 0); + + // Third repo: the LRU workspace ("a") must be evicted (killed exactly + // once) to make room — and the answer must still be correct. + let refs = h + .pool + .find_refs(&PathBuf::from("h.exe"), &sln("c"), "Sym") + .unwrap(); + assert_eq!(refs.references.len(), 1); + assert_eq!(h.spawns.load(Ordering::SeqCst), 3); + assert_eq!(h.state.kill_count.load(Ordering::SeqCst), 1); + assert_eq!(h.state.live_children.load(Ordering::SeqCst), 2); +} + +#[test] +fn resident_pool_reuses_one_workspace_per_repo() { + let h = harness(2, Duration::from_secs(600)); + for _ in 0..5 { + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + } + assert_eq!(h.spawns.load(Ordering::SeqCst), 1, "same repo = one spawn"); + assert_eq!(h.state.kill_count.load(Ordering::SeqCst), 0); +} + +#[test] +fn resident_pool_defers_kill_while_in_flight_but_still_kills() { + // The doomed path cannot be driven through the public API (a real + // in-flight request would have to overlap admission), so exercise the + // invariant directly: eviction of an idle entry kills NOW; the counter- + // then-teardown rule guarantees an in-flight entry is never killed by + // the evictor — proven here by eviction of an idle entry only. + let h = harness(1, Duration::from_secs(600)); + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("b"), "Sym") + .unwrap(); + assert_eq!(h.state.kill_count.load(Ordering::SeqCst), 1); + assert_eq!(h.state.live_children.load(Ordering::SeqCst), 1); +} + +#[test] +fn resident_pool_evicted_repo_respawns_and_answers() { + let h = harness(1, Duration::from_secs(600)); + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + // Same repo, but the single-slot pool evicted it for the second repo... + let refs = h + .pool + .find_refs(&PathBuf::from("h.exe"), &sln("b"), "Sym") + .unwrap(); + assert_eq!(refs.references.len(), 1); + // ...and coming back to the first repo must respawn and still answer. + let refs = h + .pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + assert_eq!(refs.references.len(), 1); + assert_eq!(h.spawns.load(Ordering::SeqCst), 3); +} + +#[test] +fn resident_pool_ttl_reaps_idle_workspaces_on_next_access() { + let h = harness(2, Duration::from_millis(1)); + // Spawn "a", let it go idle (TTL is 1ms), then access repo "b": the + // lazy reap on that access must kill the idle "a" workspace. + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .unwrap(); + std::thread::sleep(Duration::from_millis(5)); + h.pool + .find_refs(&PathBuf::from("h.exe"), &sln("b"), "Sym") + .unwrap(); + assert!( + h.state.kill_count.load(Ordering::SeqCst) >= 1, + "idle workspace must be reaped lazily" + ); + // Fresh spawn after reap; the answer stays correct. + let refs = h + .pool + .find_refs(&PathBuf::from("h.exe"), &sln("b"), "Sym") + .unwrap(); + assert_eq!(refs.references.len(), 1); +} + +#[test] +fn resident_pool_surfaces_client_failure_to_the_fallback() { + let h = harness(2, Duration::from_secs(600)); + h.state.fail.store(true, Ordering::SeqCst); + let err = h + .pool + .find_refs(&PathBuf::from("h.exe"), &sln("a"), "Sym") + .expect_err("a failing helper must surface an error for the one-shot fallback"); + assert!(err.to_string().contains("mock helper failed")); +} diff --git a/src/symbols/scip_parse.rs b/src/symbols/scip_parse.rs index 6c2747f2..4ac1b65b 100644 --- a/src/symbols/scip_parse.rs +++ b/src/symbols/scip_parse.rs @@ -77,7 +77,7 @@ mod roles { /// The only scip-csharp output version this parser understands (index, find-refs, batch-find-refs). /// Bump together with the helper whenever any JSON schema changes. -pub const SUPPORTED_INDEX_VERSION: &str = "1.0"; +pub const SUPPORTED_INDEX_VERSION: &str = "2.0"; // ── find-refs output format ─────────────────────────────────────── @@ -85,6 +85,11 @@ pub const SUPPORTED_INDEX_VERSION: &str = "1.0"; pub struct FindRefsResult { /// Reference locations (kind = "reference"). Does not include definitions. pub references: Vec, + /// Non-fatal problems the helper survived while resolving (project + /// compile failures, FindReferencesAsync exceptions). Non-empty means + /// `references` may be incomplete. Empty when the helper pre-dates the + /// field (serde default) or the run was clean. + pub warnings: Vec, } #[derive(Deserialize)] @@ -93,6 +98,10 @@ struct JsonFindRefsOutput { #[allow(dead_code)] symbol: String, references: Vec, + /// Absent in helper output from before warnings existed — default to + /// empty so old binaries keep parsing as "complete". + #[serde(default)] + warnings: Vec, } #[derive(Deserialize)] @@ -127,7 +136,10 @@ pub fn parse_find_refs_output(data: &[u8]) -> Result { }) .collect(); - Ok(FindRefsResult { references }) + Ok(FindRefsResult { + references, + warnings: output.warnings, + }) } /// Parse a JSON byte slice into a symbol -> references map. @@ -230,7 +242,7 @@ mod tests { #[test] fn test_parse_json_index_basic() { let json = r#"{ - "metadata": {"version": "1.0", "tool_info": "scip-csharp"}, + "metadata": {"version": "2.0", "tool_info": "scip-csharp"}, "documents": [{ "relative_path": "src/Program.cs", "occurrences": [{ @@ -263,7 +275,7 @@ mod tests { #[test] fn test_parse_json_index_empty_symbol_skipped() { let json = r#"{ - "metadata": {"version": "1.0", "tool_info": "test"}, + "metadata": {"version": "2.0", "tool_info": "test"}, "documents": [{ "relative_path": "src/A.cs", "occurrences": [{ @@ -283,7 +295,7 @@ mod tests { #[test] fn test_parse_json_index_rejects_unknown_version() { let json = r#"{ - "metadata": {"version": "2.0", "tool_info": "x"}, + "metadata": {"version": "9.9", "tool_info": "x"}, "documents": [], "external_symbols": [] }"#; @@ -294,4 +306,56 @@ mod tests { err ); } + + #[test] + fn test_parse_find_refs_output_carries_warnings_through() { + let json = r#"{ + "version": "2.0", + "symbol": "csharp Ns . V#Validate().", + "references": [{ + "file": "src/A.cs", + "start_line": 3, + "end_line": 3, + "kind": "reference" + }], + "warnings": [ + "could not compile project 'Broken' — its symbols are missing from the map", + "FindReferencesAsync failed for Validate: InvalidOperationException: boom" + ] + }"#; + + let result = parse_find_refs_output(json.as_bytes()).unwrap(); + assert_eq!(result.references.len(), 1); + assert_eq!(result.warnings.len(), 2, "warnings must survive the parse"); + assert!(result.warnings[0].contains("could not compile")); + assert!( + result.warnings[1].contains("FindReferencesAsync failed for Validate"), + "warning text must round-trip verbatim, got: {}", + result.warnings[1] + ); + } + + #[test] + fn test_parse_find_refs_output_without_warnings_field_is_complete() { + // Old helper binaries emit no warnings field: the serde default must + // parse them as EMPTY (complete), never fail. + let json = r#"{ + "version": "2.0", + "symbol": "csharp Ns . V#Validate().", + "references": [{ + "file": "src/A.cs", + "start_line": 3, + "end_line": 3, + "kind": "reference" + }] + }"#; + + let result = parse_find_refs_output(json.as_bytes()).unwrap(); + assert_eq!(result.references.len(), 1); + assert!( + result.warnings.is_empty(), + "missing field must default to complete, got: {:?}", + result.warnings + ); + } } diff --git a/src/symbols/typescript.rs b/src/symbols/typescript.rs index ca972b4a..1cc11b24 100644 --- a/src/symbols/typescript.rs +++ b/src/symbols/typescript.rs @@ -10,8 +10,8 @@ //! `scip-typescript` emits full occurrences (definitions AND references) in a //! single indexing pass. Unlike the C# adapter (`csharp.rs`), there is no lazy //! `find-refs` subprocess and no `scip_ref_cache` table: `rebuild()` populates -//! `scip_symbols` with everything up front, and `find_references()` / -//! `find_references_by_position()` only ever read LMDB. +//! `scip_symbols` with everything up front, and `resolve_query()` / +//! `find_references_for_key()` only ever read LMDB. //! //! ## Incremental rebuild //! @@ -20,21 +20,22 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use crate::lmdb_registry::TrackedEnv; use anyhow::{Context, Result}; use heed::types::{Bytes, Str}; -use heed::{Database, EnvOpenOptions}; +use heed::Database; use serde::{Deserialize, Serialize}; use super::scip_proto; -use super::{RebuildScope, RebuildSummary, SymbolIndexer, SymbolReference}; +use super::{ImpactQuery, KeyMatch, RebuildScope, RebuildSummary, SymbolIndexer, SymbolReference}; use crate::constants::{ - LANG_TYPESCRIPT, SCIP_LMDB_DEFAULT_MAP_SIZE_MB, SCIP_LMDB_MAP_SIZE_MB_ENV, - SCIP_POSITION_DB_NAME, SCIP_SIMPLE_NAMES_DB_NAME, SCIP_SYMBOLS_DB_NAME, - SCIP_TYPESCRIPT_HELPER_ENV, SCIP_TYPESCRIPT_REBUILD_TIMESTAMP_KEY, + LANG_TYPESCRIPT, SCIP_POSITION_DB_NAME, SCIP_SIMPLE_NAMES_DB_NAME, SCIP_SYMBOLS_DB_NAME, + SCIP_TYPESCRIPT_HEAD_SHA_KEY, SCIP_TYPESCRIPT_HELPER_ENV, SCIP_TYPESCRIPT_INDEX_WARNINGS_KEY, + SCIP_TYPESCRIPT_REBUILD_TIMESTAMP_KEY, }; // ── Constants ───────────────────────────────────────────────────── @@ -45,7 +46,7 @@ const SCIP_DB_NAME: &str = SCIP_SYMBOLS_DB_NAME; /// LMDB database name for the rebuild timestamp / metadata table. /// Shares the physical table with the C# adapter, but keys are namespaced /// per-language (see `SCIP_TYPESCRIPT_REBUILD_TIMESTAMP_KEY`). -const SCIP_META_DB_NAME: &str = "scip_meta"; +const SCIP_META_DB_NAME: &str = crate::constants::SCIP_META_DB_NAME; /// LMDB database name for the position-to-symbols index. const SCIP_POS_DB_NAME: &str = SCIP_POSITION_DB_NAME; @@ -56,6 +57,13 @@ const SCIP_NAMES_DB_NAME: &str = SCIP_SIMPLE_NAMES_DB_NAME; /// Key in the meta database storing the last rebuild timestamp for TypeScript. const META_REBUILD_TS: &str = SCIP_TYPESCRIPT_REBUILD_TIMESTAMP_KEY; +/// Key in the meta database storing the git HEAD sha the index was built for. +const META_HEAD_SHA: &str = SCIP_TYPESCRIPT_HEAD_SHA_KEY; + +/// Key in the meta database storing the index-completeness warnings (a JSON +/// array) from the last `scip-typescript` run. Empty array = complete. +const META_INDEX_WARNINGS: &str = SCIP_TYPESCRIPT_INDEX_WARNINGS_KEY; + /// Key in the meta database storing the count of indexed symbols. const META_SYMBOL_COUNT: &str = "symbol_count:typescript"; @@ -263,43 +271,26 @@ impl TypeScriptSymbolIndexer { } } - /// Open or create the SCIP LMDB environment for a given repo database path. - /// Shares the same on-disk tables as the C# adapter (`db_path/scip/`), - /// distinguished by namespaced keys/values where needed. - fn open_scip_env(&self, db_path: &Path) -> Result { - let scip_dir = db_path.join("scip"); - std::fs::create_dir_all(&scip_dir) - .with_context(|| format!("Failed to create SCIP directory: {}", scip_dir.display()))?; - - let map_size_mb = std::env::var(SCIP_LMDB_MAP_SIZE_MB_ENV) - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(SCIP_LMDB_DEFAULT_MAP_SIZE_MB); - let mut opts = EnvOpenOptions::new(); - opts.map_size(map_size_mb * 1024 * 1024).max_dbs(10); - // SAFETY: `NO_TLS` only changes reader-slot tracking. See `BASE_ENV_FLAGS`. - unsafe { opts.flags(crate::lmdb_registry::BASE_ENV_FLAGS) }; - let env = - unsafe { TrackedEnv::open(&opts, &scip_dir, &format!("SCIP({})", db_path.display()))? }; - - let mut wtxn = env.write_txn()?; - env.create_database::(&mut wtxn, Some(SCIP_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_META_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_POS_DB_NAME))?; - env.create_database::(&mut wtxn, Some(SCIP_NAMES_DB_NAME))?; - wtxn.commit()?; - - Ok(env) + /// Open the shared SCIP LMDB environment for a given repo database path. + /// + /// Delegates to [`crate::symbols::get_shared_scip_env`] — TS shares the + /// C# adapter's `db_path/scip/` directory (distinguished by namespaced + /// keys), so it must also share its environment: two adapters opening the + /// same directory concurrently would trip the double-open guard. + fn open_scip_env(&self, db_path: &Path) -> Result> { + crate::symbols::get_shared_scip_env(db_path) } /// Invoke `scip-typescript index` against `project_root`, writing the SCIP - /// protobuf index to `output_path`. + /// protobuf index to `output_path`. Returns the completeness warning when + /// the run exited non-zero (partial output is acceptable, mirroring the + /// C# adapter) — the caller persists it so the partial index stays honest. fn invoke_index_helper( &self, invocation: &HelperInvocation, project_root: &Path, output_path: &Path, - ) -> Result<()> { + ) -> Result> { let mut cmd = match invocation { HelperInvocation::Direct(path) => Command::new(path), HelperInvocation::Npx => { @@ -360,50 +351,62 @@ impl TypeScriptSymbolIndexer { } if !output.status.success() { - tracing::warn!( - "scip-typescript exited with {} for {}", - output.status, + let warning = format!( + "scip-typescript exited with {} for {} — index may be incomplete", + super::exit_status_text(&output.status), project_root.display() ); - // Don't bail — partial output is acceptable, mirroring the C# adapter. + tracing::warn!("{warning}"); + // Don't bail — partial output is acceptable, mirroring the C# + // adapter — but the warning travels to the caller, which + // persists it so the partial index can never pass for complete. + return Ok(Some(warning)); } - Ok(()) + Ok(None) } - /// Resolve a user-supplied symbol query to a canonical SCIP symbol key. + /// Resolve a user-supplied symbol query to canonical SCIP key(s). /// Exact match first, then fuzzy match via the simple-name index. - fn resolve_canonical_key(&self, env: &TrackedEnv, symbol: &str) -> Result> { + /// + /// Several fuzzy matches come back as `KeyMatch::Ambiguous` — the old + /// behaviour silently picked the shortest candidate, which hid + /// overloads from the caller. + fn resolve_name_key(&self, env: &TrackedEnv, symbol: &str) -> Result { let rtxn = env.read_txn()?; let symbols_db: Database = match env.open_database(&rtxn, Some(SCIP_DB_NAME))? { Some(db) => db, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; if symbols_db.get(&rtxn, symbol)?.is_some() { - return Ok(Some(symbol.to_string())); + return Ok(KeyMatch::Resolved(symbol.to_string())); } let simple_names_db: Database = match env.open_database(&rtxn, Some(SCIP_NAMES_DB_NAME))? { Some(db) => db, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; let simple = extract_simple_name(symbol); let candidates: Vec = match simple_names_db.get(&rtxn, &simple as &str)? { Some(b) => deserialize_keys_v1(b)?, - None => return Ok(None), + None => return Ok(KeyMatch::NotFound), }; - let chosen = candidates - .iter() + let mut matches: Vec = candidates + .into_iter() .filter(|k| fuzzy_symbol_match(symbol, k)) - .min_by_key(|k| k.len()) - .cloned(); - - Ok(chosen) + .collect(); + matches.sort(); + matches.dedup(); + Ok(match matches.len() { + 0 => KeyMatch::NotFound, + 1 => KeyMatch::Resolved(matches.pop().expect("len checked")), + _ => KeyMatch::Ambiguous(matches), + }) } } @@ -473,7 +476,7 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { } let _output_guard = TempFileGuard(output_path.clone()); - self.invoke_index_helper(&invocation, repo_path, &output_path)?; + let index_warning = self.invoke_index_helper(&invocation, repo_path, &output_path)?; let index_data = std::fs::read(&output_path) .with_context(|| format!("Failed to read SCIP index at {}", output_path.display()))?; @@ -575,6 +578,23 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { META_SYMBOL_COUNT, total_symbols.to_string().as_str(), )?; + // Fingerprint the index: which HEAD it was built for (language- + // prefixed key — the meta table is shared with the C# adapter). + // Skipped when git is unreadable, same policy as the C# adapter. + if let Some(sha) = super::current_git_head(repo_path) { + meta_db.put(&mut wtxn, META_HEAD_SHA, sha.as_str())?; + } + // Persist index completeness as a JSON array. ALWAYS written — also + // on a clean run, as an empty array — so a successful reindex clears + // stale warnings instead of leaving a fixed index claiming partiality. + let index_warnings: Vec = index_warning.into_iter().collect(); + meta_db.put( + &mut wtxn, + META_INDEX_WARNINGS, + serde_json::to_string(&index_warnings) + .context("Failed to serialize TypeScript index warnings")? + .as_str(), + )?; wtxn.commit()?; @@ -594,16 +614,65 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { }) } - fn find_references(&self, db_path: &Path, symbol: &str) -> Result> { - let env = self.open_scip_env(db_path)?; - - let canonical = match self.resolve_canonical_key(&env, symbol)? { - Some(k) => k, - None => { - tracing::debug!("Symbol '{}' not found in TypeScript index", symbol); - return Ok(vec![]); + fn resolve_query(&self, db_path: &Path, query: &ImpactQuery) -> Result { + match query { + ImpactQuery::ExactKey(key) => { + // Explicit selection: presence check only, no fuzzy + // fallback. A key that is not in the index is NotFound — + // the handler turns that into a loud failure, never a + // guess at a near-miss symbol. + let env = self.open_scip_env(db_path)?; + let rtxn = env.read_txn()?; + let present = match env.open_database::(&rtxn, Some(SCIP_DB_NAME))? { + Some(db) => db.get(&rtxn, key as &str)?.is_some(), + None => false, + }; + Ok(if present { + KeyMatch::Resolved(key.clone()) + } else { + KeyMatch::NotFound + }) } - }; + ImpactQuery::Name(name) => { + let env = self.open_scip_env(db_path)?; + self.resolve_name_key(&env, name) + } + ImpactQuery::Position { file, line } => { + let env = self.open_scip_env(db_path)?; + let rtxn = env.read_txn()?; + + let positions_db: Database = env + .open_database(&rtxn, Some(SCIP_POS_DB_NAME))? + .ok_or_else(|| { + anyhow::anyhow!("Position index not found. Run a rebuild first.") + })?; + + let pos_key = format!("{}:{}", file.to_string_lossy().replace('\\', "/"), line); + + let mut candidates: Vec = match positions_db.get(&rtxn, &pos_key as &str)? { + Some(b) => deserialize_keys_v1(b)?, + None => return Ok(KeyMatch::NotFound), + }; + candidates.sort(); + candidates.dedup(); + + // Several symbols on one line are ambiguity, not a licence + // to pick the shortest. + Ok(match candidates.len() { + 0 => KeyMatch::NotFound, + 1 => KeyMatch::Resolved(candidates.pop().expect("len checked")), + _ => KeyMatch::Ambiguous(candidates), + }) + } + } + } + + fn find_references_for_key( + &self, + db_path: &Path, + canonical_key: &str, + ) -> Result> { + let env = self.open_scip_env(db_path)?; let rtxn = env.read_txn()?; let symbols_db: Database = match env.open_database(&rtxn, Some(SCIP_DB_NAME))? { @@ -611,7 +680,7 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { None => return Ok(vec![]), }; - let stored = match symbols_db.get(&rtxn, &canonical)? { + let stored = match symbols_db.get(&rtxn, canonical_key)? { Some(bytes) => deserialize_refs(bytes)?, None => return Ok(vec![]), }; @@ -627,34 +696,25 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { .collect()) } - fn find_references_by_position( - &self, - db_path: &Path, - file: &Path, - line: u32, - ) -> Result> { - let env = self.open_scip_env(db_path)?; - let rtxn = env.read_txn()?; - - let positions_db: Database = env - .open_database(&rtxn, Some(SCIP_POS_DB_NAME))? - .ok_or_else(|| anyhow::anyhow!("Position index not found. Run a rebuild first."))?; - - let pos_key = format!("{}:{}", file.to_string_lossy().replace('\\', "/"), line); - - let candidate_keys: Vec = match positions_db.get(&rtxn, &pos_key as &str)? { - Some(b) => deserialize_keys_v1(b)?, - None => return Ok(vec![]), + fn lookup_warnings(&self, db_path: &Path, _canonical: &str) -> Vec { + // Plain LMDB read — never a helper invocation. The meta entry is + // per-INDEX, not per-symbol (scip-typescript has no per-symbol + // resolution), so every canonical key of this index reports the + // same warnings. Absence (pre-warnings indexes) reads as complete. + let Some(env) = self.open_scip_env(db_path).ok() else { + return Vec::new(); }; - - let chosen = candidate_keys.iter().min_by_key(|k| k.len()).cloned(); - drop(rtxn); - drop(env); - - match chosen { - Some(k) => self.find_references(db_path, &k), - None => Ok(vec![]), - } + let Ok(rtxn) = env.read_txn() else { + return Vec::new(); + }; + let raw = match env.open_database::(&rtxn, Some(SCIP_META_DB_NAME)) { + Ok(Some(meta)) => match meta.get(&rtxn, META_INDEX_WARNINGS) { + Ok(Some(raw)) => raw, + _ => return Vec::new(), + }, + _ => return Vec::new(), + }; + serde_json::from_str(raw).unwrap_or_default() } fn index_age(&self, db_path: &Path) -> u64 { @@ -690,6 +750,18 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { now.saturating_sub(stored_ts) } + fn index_head_sha(&self, db_path: &Path) -> Option { + let env = self.open_scip_env(db_path).ok()?; + let rtxn = env.read_txn().ok()?; + let meta_db: Database = env + .open_database(&rtxn, Some(SCIP_META_DB_NAME)) + .ok() + .flatten()?; + let sha = meta_db.get(&rtxn, META_HEAD_SHA).ok().flatten()?; + let sha = sha.trim().to_string(); + (!sha.is_empty()).then_some(sha) + } + fn has_index(&self, db_path: &Path) -> bool { let scip_dir = db_path.join("scip"); if !scip_dir.exists() { @@ -777,3 +849,9 @@ mod tests { // `tmp` dropped at end of scope → dir removed even on panic. } } + +/// Ambiguity-resolution tests for `resolve_query` (hand-populated LMDB — +/// no helper). Sibling `_tests.rs` file per repo convention. +#[cfg(test)] +#[path = "typescript_tests.rs"] +mod typescript_tests; diff --git a/src/symbols/typescript_tests.rs b/src/symbols/typescript_tests.rs new file mode 100644 index 00000000..890a4187 --- /dev/null +++ b/src/symbols/typescript_tests.rs @@ -0,0 +1,330 @@ +//! `resolve_query` semantics for the TypeScript adapter (hand-populated +//! LMDB — no `scip-typescript` helper). Mirrors the C# fixture: two +//! `add` "overloads" sharing a simple name, one unique `mul`. + +use super::*; + +/// Writes scip_symbols / scip_simple_names / scip_positions directly and +/// returns the three canonical keys it stored. +fn populate_ambiguity_fixture(db_path: &Path) -> (String, String, String) { + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + + let add1 = "scip-typescript npm mypkg 1.0.0 `src/math.ts`/add().".to_string(); + let add2 = "scip-typescript npm mypkg 1.0.0 `src/math.ts`/add(`x`: number).".to_string(); + let mul = "scip-typescript npm mypkg 1.0.0 `src/math.ts`/mul().".to_string(); + + let symbols: Database = env + .open_database(&wtxn, Some(SCIP_DB_NAME)) + .unwrap() + .unwrap(); + for key in [&add1, &add2, &mul] { + let refs = serialize_refs(&[StoredReference { + file: PathBuf::from("src/math.ts"), + start_line: 1, + end_line: 1, + kind: "definition".into(), + }]) + .unwrap(); + symbols.put(&mut wtxn, key.as_str(), &refs).unwrap(); + } + + let names: Database = env + .open_database(&wtxn, Some(SCIP_NAMES_DB_NAME)) + .unwrap() + .unwrap(); + // Stored deliberately out of order: resolution must sort. + names + .put( + &mut wtxn, + "add", + &serialize_keys_v1(&[add2.clone(), add1.clone()]).unwrap(), + ) + .unwrap(); + names + .put( + &mut wtxn, + "mul", + &serialize_keys_v1(std::slice::from_ref(&mul)).unwrap(), + ) + .unwrap(); + + let positions: Database = env + .open_database(&wtxn, Some(SCIP_POS_DB_NAME)) + .unwrap() + .unwrap(); + positions + .put( + &mut wtxn, + "src/math.ts:10", + &serialize_keys_v1(&[add1.clone(), add2.clone()]).unwrap(), + ) + .unwrap(); + positions + .put( + &mut wtxn, + "src/math.ts:20", + &serialize_keys_v1(std::slice::from_ref(&mul)).unwrap(), + ) + .unwrap(); + + wtxn.commit().unwrap(); + (add1, add2, mul) +} + +#[test] +fn resolve_name_unique_fuzzy_resolves_the_single_candidate() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (_a1, _a2, mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::Name("mul".into())) + .unwrap(), + KeyMatch::Resolved(mul) + ); +} + +#[test] +fn resolve_name_overloads_come_back_ambiguous_and_sorted() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (a1, a2, _mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + // The pre-fix behaviour silently picked the shortest key here and + // answered about the wrong overload. + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::Name("add".into())) + .unwrap(), + KeyMatch::Ambiguous(vec![a1, a2]) + ); +} + +#[test] +fn resolve_exact_key_is_verbatim_and_never_fuzzy() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (a1, _a2, _mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + assert_eq!( + indexer + .resolve_query(&db, &ImpactQuery::ExactKey(a1.clone())) + .unwrap(), + KeyMatch::Resolved(a1) + ); + // A key that is only a prefix of a stored one must miss. + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::ExactKey("scip-typescript npm mypkg 1.0.0 `src/math.ts`/add".into()) + ) + .unwrap(), + KeyMatch::NotFound + ); +} + +#[test] +fn resolve_position_single_resolves_two_symbols_are_ambiguous() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (a1, a2, mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/math.ts"), + line: 20 + } + ) + .unwrap(), + KeyMatch::Resolved(mul) + ); + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/math.ts"), + line: 10 + } + ) + .unwrap(), + KeyMatch::Ambiguous(vec![a1, a2]) + ); + assert_eq!( + indexer + .resolve_query( + &db, + &ImpactQuery::Position { + file: PathBuf::from("src/math.ts"), + line: 99 + } + ) + .unwrap(), + KeyMatch::NotFound + ); +} + +#[test] +fn references_for_key_returns_stored_definitions() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (a1, _a2, _mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + let refs = indexer.find_references_for_key(&db, &a1).unwrap(); + assert_eq!(refs.len(), 1, "definitions only, got {refs:?}"); + assert_eq!(refs[0].kind, "definition"); + assert_eq!(refs[0].file, PathBuf::from("src/math.ts")); +} + +#[test] +fn lookup_warnings_reads_the_index_warnings_meta_entry() { + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let (a1, _a2, _mul) = populate_ambiguity_fixture(&db); + let indexer = TypeScriptSymbolIndexer::new(); + + // No entry yet (pre-warnings index) → empty = complete. + assert!(indexer.lookup_warnings(&db, &a1).is_empty()); + + // Hand-populate the meta entry exactly the way the rebuild flow writes + // it: a JSON array under the namespaced meta key. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: Database = env + .open_database(&wtxn, Some(SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + meta.put( + &mut wtxn, + META_INDEX_WARNINGS, + r#"["scip-typescript exited with 1 for /repo — index may be incomplete"]"#, + ) + .unwrap(); + wtxn.commit().unwrap(); + + let warnings = indexer.lookup_warnings(&db, &a1); + assert_eq!(warnings.len(), 1, "the stored warning must be read back"); + assert!( + warnings[0].contains("scip-typescript exited with 1"), + "warning text must round-trip, got: {}", + warnings[0] + ); + + // The entry is per-index, not per-symbol: every key reports it. + assert_eq!( + indexer + .lookup_warnings(&db, "scip-typescript npm mypkg 1.0.0 `src/other.ts`/x().") + .len(), + 1 + ); + + // An empty array (what a clean reindex writes) must read as complete. + let mut wtxn = env.write_txn().unwrap(); + let meta: Database = env + .open_database(&wtxn, Some(SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + meta.put(&mut wtxn, META_INDEX_WARNINGS, "[]").unwrap(); + wtxn.commit().unwrap(); + assert!( + indexer.lookup_warnings(&db, &a1).is_empty(), + "an empty warnings array must mean complete" + ); +} + +// ── B3 warnings: the rebuild's meta-write site, driven for real ────── +// +// The test above hand-writes the META_INDEX_WARNINGS entry (consumer +// half). This one drives the producer: a helper that exits non-zero, so +// the rebuild flow must persist the completeness warning itself. + +/// A fake `scip-typescript` with the partial-output contract: exits +/// non-zero AFTER writing the `--output` file, so the rebuild must both +/// capture the warning and parse the (here empty — a valid default index) +/// output. `resolve_helper` only checks `is_file()`, so a shell script is +/// accepted as the env-var helper. +fn write_failing_helper(dir: &Path) -> PathBuf { + let (path, script) = if cfg!(windows) { + ( + dir.join("fake-scip-typescript.cmd"), + "@echo off\r\ntype nul > \"%~3\"\r\nexit /b 3\r\n".to_string(), + ) + } else { + ( + dir.join("fake-scip-typescript"), + "#!/bin/sh\n: > \"$3\"\nexit 3\n".to_string(), + ) + }; + std::fs::write(&path, script).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + path +} + +#[test] +#[serial_test::serial] +fn rebuild_persists_the_nonzero_exit_warning_into_the_meta_table() { + let dir = tempfile::TempDir::new().unwrap(); + let repo = dir.path().join("repo"); + let db = dir.path().join("db"); + std::fs::create_dir_all(&repo).unwrap(); + std::fs::write(repo.join("tsconfig.json"), "{}").unwrap(); + + let helper_bin = dir.path().join("helper-bin"); + std::fs::create_dir(&helper_bin).unwrap(); + let helper = write_failing_helper(&helper_bin); + let _guard = crate::testing::EnvRestore::set(&[( + SCIP_TYPESCRIPT_HELPER_ENV, + helper.to_string_lossy().as_ref(), + )]); + + let indexer = TypeScriptSymbolIndexer::new(); + let any_key = "scip-typescript npm mypkg 1.0.0 `src/x.ts`/f()."; + assert!(indexer.lookup_warnings(&db, any_key).is_empty()); + + // Non-zero helper exit must NOT fail the rebuild — partial output is + // acceptable — but the warning must ride into the meta table. + let summary = indexer + .rebuild(&repo, &db, RebuildScope::Full) + .expect("a failing helper must still complete the rebuild"); + assert_eq!( + summary.symbols_indexed, 0, + "the fake helper wrote an empty index, nothing else" + ); + + // THE PRODUCER ASSERTION: the warning from the non-zero exit was + // persisted by the rebuild's META_INDEX_WARNINGS write. + let warnings = indexer.lookup_warnings(&db, any_key); + assert_eq!( + warnings.len(), + 1, + "the rebuild must persist the non-zero-exit warning" + ); + assert!( + warnings[0].contains("exit code: 3"), + "warning text must name the exit status, got: {}", + warnings[0] + ); +} + +#[test] +fn exit_status_text_is_platform_stable() { + // ExitStatus's Display prints "exit code: N" on Windows but + // "exit status: N" on Unix — persisted warning text (and the test + // above) must not drift with the host OS. Pin the helper on both. + #[cfg(unix)] + let status = std::os::unix::process::ExitStatusExt::from_raw(3 << 8); + #[cfg(windows)] + let status = std::os::windows::process::ExitStatusExt::from_raw(3); + assert_eq!(crate::symbols::exit_status_text(&status), "exit code: 3"); +} diff --git a/src/testing.rs b/src/testing.rs new file mode 100644 index 00000000..6caf397d --- /dev/null +++ b/src/testing.rs @@ -0,0 +1,54 @@ +//! Shared test-only helpers. Compiled into the crate under `#[cfg(test)]` +//! only, so none of this ships in release builds. + +/// RAII guard that sets the given env vars and restores their previous +/// values (including "was unset") when dropped. +/// +/// Tests that mutate process-global env vars must BOTH use this guard AND +/// be annotated `#[serial]` (serial_test crate): the guard makes the +/// mutation panic-safe (a failing assertion can no longer leak a stale +/// value into every later test), and `#[serial]` serializes them against +/// the other tests in the same cargo-test process that read the same vars +/// -- the doctor tests read CODESEARCH_REPOS_CONFIG on code paths that +/// race the remove_order_tests writes, which is the flake class this +/// pair was introduced to close. See AGENTS.md "Notes for agents". +pub struct EnvRestore { + saved: Vec<(&'static str, Option)>, +} + +impl EnvRestore { + /// Snapshot the current values of `vars`, then set them. Restores on drop. + pub fn set(vars: &[(&'static str, &str)]) -> Self { + let saved = vars + .iter() + .map(|(k, _)| (*k, std::env::var(k).ok())) + .collect(); + for (k, v) in vars { + std::env::set_var(k, v); + } + Self { saved } + } + + /// Snapshot the current values of `vars` (including "was unset"), then + /// REMOVE them all. Restores on drop. Complements [`EnvRestore::set`] + /// for the "variable must be absent" cases (e.g. asserting the default + /// fallback of an env-overridable knob). + pub fn remove(vars: &[&'static str]) -> Self { + let saved = vars.iter().map(|k| (*k, std::env::var(k).ok())).collect(); + for k in vars { + std::env::remove_var(k); + } + Self { saved } + } +} + +impl Drop for EnvRestore { + fn drop(&mut self) { + for (k, prev) in &self.saved { + match prev { + Some(v) => std::env::set_var(k, v), + None => std::env::remove_var(k), + } + } + } +} diff --git a/src/vectordb/store.rs b/src/vectordb/store.rs index 968009c0..63643bba 100644 --- a/src/vectordb/store.rs +++ b/src/vectordb/store.rs @@ -384,12 +384,48 @@ pub struct VectorStore { env: TrackedEnv, vectors: ArroyDatabase, chunks: Database, SerdeBincode>, + /// Persisted high-water mark of chunk ids ever handed out ("meta" DB, + /// key [`META_KEY_ID_HWM`]). `None` only in read-only mode on a legacy + /// store created before the mark existed. + /// + /// Without this, `next_id` is derived from `chunks.last()` on every open, + /// so deleting the chunks holding the highest ids lowers `max_key` and the + /// next reopen hands those ids to unrelated content — `get_chunk(old_id)` + /// then silently returns the wrong file. The mark never decreases on + /// delete; deleted ids stay dead forever (safe `Ok(None)` misses). + id_hwm_db: Option>>, next_id: u32, dimensions: usize, indexed: bool, pub map_size_mb: usize, } +/// Key in the "meta" database holding the highest chunk id ever assigned. +const META_KEY_ID_HWM: &str = "id_hwm"; + +/// Derive `next_id` so ids are NEVER reused across reopens. +/// +/// Takes the max of (highest live key + 1) and (persisted high-water mark + 1): +/// - live keys alone regress when top-of-range chunks are deleted; +/// - the mark alone is absent on legacy stores (falls back to live keys — +/// pre-mark behaviour, unchanged until the first write persists the mark). +/// +/// A full rebuild wipes the DB and the mark with it, which is correct: a new +/// generation may restart at 0, and stale references then fail as `Ok(None)` +/// (safe miss) instead of resolving to unrelated content. +fn next_id_from( + chunks: &Database, SerdeBincode>, + hwm: Option, + txn: &heed::RoTxn, +) -> Result { + let from_live = match chunks.last(txn)? { + Some((max_key, _)) => max_key + 1, + None => 0, + }; + let from_mark = hwm.map(|h| h.saturating_add(1)).unwrap_or(0); + Ok(from_live.max(from_mark)) +} + /// Lightweight chunk metadata used for file-outline style navigation. #[derive(Debug, Clone)] pub struct ChunkMeta { @@ -451,14 +487,17 @@ impl VectorStore { let vectors: ArroyDatabase = env.create_database(&mut wtxn, Some("vectors"))?; let chunks: Database, SerdeBincode> = env.create_database(&mut wtxn, Some("chunks"))?; - - // Get the next ID from the maximum existing key + 1 - // Using len() is wrong after delete+insert cycles: deleted IDs create gaps - // so len() < max_key + 1, causing ID collisions on re-open - let next_id = match chunks.last(&wtxn)? { - Some((max_key, _)) => max_key + 1, - None => 0, - }; + let id_hwm_db: Database> = + env.create_database(&mut wtxn, Some("meta"))?; + + // Get the next ID from the maximum existing key + 1 and the persisted + // high-water mark, whichever is higher — see `next_id_from`. Using + // len() is wrong after delete+insert cycles: deleted IDs create gaps + // so len() < max_key + 1, causing ID collisions on re-open; using + // max_key alone is wrong after TOP-OF-RANGE deletes, which lower + // max_key and would hand those ids to unrelated new content. + let hwm: Option = id_hwm_db.get(&wtxn, META_KEY_ID_HWM)?; + let next_id = next_id_from(&chunks, hwm, &wtxn)?; wtxn.commit()?; @@ -485,6 +524,7 @@ impl VectorStore { env, vectors, chunks, + id_hwm_db: Some(id_hwm_db), next_id, dimensions, indexed, @@ -547,13 +587,20 @@ impl VectorStore { let chunks: Database, SerdeBincode> = env .open_database(&rtxn, Some("chunks"))? .ok_or_else(|| anyhow::anyhow!("chunks database not found"))?; - - // Get the next ID from the maximum existing key + 1 - // Using len() is wrong after delete+insert cycles: deleted IDs create gaps - let next_id = match chunks.last(&rtxn)? { - Some((max_key, _)) => max_key + 1, - None => 0, + // The mark DB may be absent on legacy stores (created before ids were + // made monotonic) — `None` then, and `next_id_from` falls back to the + // live-keys derivation. Read-only never inserts, so the mark is only + // informational here anyway. + let id_hwm_db: Option>> = + env.open_database(&rtxn, Some("meta"))?; + + // Get the next ID from the maximum existing key + 1 and the persisted + // high-water mark, whichever is higher — see `next_id_from`. + let hwm: Option = match &id_hwm_db { + Some(db) => db.get(&rtxn, META_KEY_ID_HWM)?, + None => None, }; + let next_id = next_id_from(&chunks, hwm, &rtxn)?; // Check if database is already indexed let indexed = if next_id > 0 { @@ -587,6 +634,7 @@ impl VectorStore { env, vectors, chunks, + id_hwm_db, next_id, dimensions, indexed, @@ -687,6 +735,11 @@ impl VectorStore { self.next_id += 1; } + // Same-transaction mark persist as in insert_chunks_with_ids_impl. + if let Some(db) = &self.id_hwm_db { + db.put(&mut wtxn, META_KEY_ID_HWM, &(self.next_id - 1))?; + } + wtxn.commit()?; // Mark as not indexed (need to rebuild index after inserts) @@ -1032,6 +1085,15 @@ impl VectorStore { self.next_id += 1; } + // Persist the high-water mark in the SAME transaction as the data: + // if this txn aborts, neither the chunks nor the mark land, so the + // mark can never claim ids that were not actually assigned. On + // abort the in-memory next_id may have advanced past the persisted + // mark — that only wastes ids (gaps), it can never reuse one. + if let Some(db) = &self.id_hwm_db { + db.put(&mut wtxn, META_KEY_ID_HWM, &(self.next_id - 1))?; + } + wtxn.commit()?; self.indexed = false; @@ -1050,6 +1112,14 @@ impl VectorStore { self.chunks.clear(&mut wtxn)?; self.vectors.clear(&mut wtxn)?; + // A deliberate wipe starts a new id generation: drop the high-water + // mark with the data so the counter may restart at 0. Stale references + // into the wiped generation then fail as `Ok(None)` (safe miss) — + // they can never resolve to the new generation's unrelated content. + if let Some(db) = &self.id_hwm_db { + db.delete(&mut wtxn, META_KEY_ID_HWM)?; + } + wtxn.commit()?; self.next_id = 0; @@ -1568,4 +1638,199 @@ mod tests { assert!(emb.is_some()); assert_eq!(emb.unwrap().len(), 4); } + + /// Helper: a 1-chunk insert carrying a distinguishing path, returning the + /// id assigned to it. + fn insert_one(store: &mut VectorStore, path: &str) -> u32 { + let ids = store + .insert_chunks_with_ids(vec![EmbeddedChunk::new( + Chunk::new( + format!("fn {path}() {{}}"), + 0, + 1, + ChunkKind::Function, + path.to_string(), + ), + vec![1.0, 0.0, 0.0, 0.0], + )]) + .unwrap(); + assert_eq!(ids.len(), 1); + ids[0] + } + + /// Deleting the chunks that hold the HIGHEST ids must not let a reopen + /// hand those ids to new content. Pre-mark behaviour recomputed + /// `next_id = max_key + 1` on every open, so the delete lowered max_key + /// and the next insert silently reused a dead id — `get_chunk(old_id)` + /// then returned the WRONG file with no error (the custom-kb + /// wrong-file-resolution defect class, todo #51). + #[test] + fn reopen_after_top_of_range_delete_does_not_reuse_ids() { + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("hwm-top.db"); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen1/a.rs"), 0); + assert_eq!(insert_one(&mut store, "gen1/b.rs"), 1); + + // Delete the top-of-range chunk (id 1) — lowers max_key to 0. + assert_eq!(store.delete_chunks(&[1]).unwrap(), 1); + drop(store); + + // Reopen: next_id must come from the persisted high-water mark (1), + // NOT from the lowered max_key (0). The new chunk gets id 2. + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen2/c.rs"), 2); + + // The deleted id stays dead: a safe miss, never unrelated content. + let stale = store.get_chunk(1).unwrap(); + assert!(stale.is_none(), "deleted id 1 must stay dead"); + // And it did not alias the new content either. + assert_eq!(store.get_chunk(2).unwrap().unwrap().path, "gen2/c.rs"); + } + + /// The sharpest variant: delete EVERYTHING. Live keys are then empty, so + /// the legacy derivation would restart at id 0 and hand it to unrelated + /// new content. The mark must keep the counter past every dead id. + /// (Custom-kb routinely hits delete+add via renames and repo rewrites.) + #[test] + fn reopen_after_full_delete_never_restarts_from_zero() { + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("hwm-full.db"); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen1/a.rs"), 0); + assert_eq!(insert_one(&mut store, "gen1/b.rs"), 1); + assert_eq!(insert_one(&mut store, "gen1/c.rs"), 2); + + assert_eq!(store.delete_chunks(&[0, 1, 2]).unwrap(), 3); + drop(store); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen2/d.rs"), 3); + for dead in 0..3 { + assert!( + store.get_chunk(dead).unwrap().is_none(), + "deleted id {dead} must stay dead" + ); + } + } + + /// A deliberate `clear()` wipes the mark with the data: the next + /// generation may restart at 0, and old references miss safely. + #[test] + fn clear_resets_the_id_generation() { + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("hwm-clear.db"); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen1/a.rs"), 0); + store.clear().unwrap(); + drop(store); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen2/b.rs"), 0); + } + + /// Legacy-store compat: a store whose "meta" DB carries no mark (written + /// by pre-mark code) opens fine and derives next_id from live keys only — + /// behaviour is unchanged until the first write persists the mark. + #[test] + fn reopen_without_mark_falls_back_to_live_keys() { + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("hwm-legacy.db"); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(insert_one(&mut store, "gen1/a.rs"), 0); + assert_eq!(insert_one(&mut store, "gen1/b.rs"), 1); + + // Simulate a legacy store: strip the mark, keep the data. + { + let mut wtxn = store.env.write_txn().unwrap(); + store + .id_hwm_db + .as_ref() + .unwrap() + .delete(&mut wtxn, META_KEY_ID_HWM) + .unwrap(); + wtxn.commit().unwrap(); + } + drop(store); + + let mut store = VectorStore::new(&db_path, 4).unwrap(); + // No mark, max_key = 1 → legacy derivation: next id is 2. (With the + // top chunk deleted this WOULD reuse id 1 — that is the documented, + // unchanged legacy risk for stores written before the mark existed.) + assert_eq!(insert_one(&mut store, "gen2/c.rs"), 2); + } + + // === cross-generation chunk-id drift (mechanism repro → FIXED) === + // + // History: ids were autoincrement (`next_id = max_key + 1` recomputed on + // every open), so deleting the chunks holding the HIGHEST ids lowered + // `max_key` and the next reopen handed those ids to unrelated content — + // `get_chunk(old_id)` returned the wrong file with no error. On the cloud + // peer every scale-to-zero wake replays custom-kb's incremental git + // history on a restored snapshot (delete + re-insert at the top of the + // range is routine there), which is why this mattered for chunk-id + // stability across cold starts. See develterf_dlwr/todo#51. + // + // The original repro on `fix/custom-kb-chunk-id-drift` + // (`reopen_after_top_of_range_delete_reassigns_ids_to_new_content`) + // asserted the OLD reassignment behaviour; it is superseded by the + // high-water-mark tests above (`reopen_after_top_of_range_delete_does_ + // not_reuse_ids`, `reopen_after_full_delete_never_restarts_from_zero`) + // which pin the FIXED behaviour. The boundary control below is kept + // verbatim: low-range deletes were always safe and must stay safe. + + fn drift_chunk(path: &str, content: &str, id: usize) -> EmbeddedChunk { + EmbeddedChunk::new( + Chunk::new( + content.to_string(), + id, + id, + ChunkKind::Other, + path.to_string(), + ), + vec![1.0, 0.0, 0.0, 0.0], + ) + } + + #[test] + fn reopen_after_low_range_delete_keeps_remaining_ids_stable() { + // Boundary control: deleting BELOW the top of the range leaves + // max_key untouched, so surviving ids stay stable across reopens and + // new inserts never collide with them. Under the high-water mark this + // holds trivially (the mark only ever raises next_id) — the test + // pins that the mark did not CHANGE this always-safe case. + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("drift-low.db"); + + { + let mut store = VectorStore::new(&db_path, 4).unwrap(); + store + .insert_chunks_with_ids(vec![ + drift_chunk("a.md", "content A0", 0), + drift_chunk("a.md", "content A1", 1), + drift_chunk("b.md", "content B0", 2), + drift_chunk("b.md", "content B1", 3), + ]) + .unwrap(); + // A (ids 0,1) deleted; B keeps the top of the range. + store.delete_chunks(&[0, 1]).unwrap(); + } + + let mut store2 = VectorStore::new(&db_path, 4).unwrap(); + assert_eq!(store2.next_id, 4, "max_key (B's id 3) keeps next_id at 4"); + + // B's ids still resolve to B after the reopen. + let chunk = store2.get_chunk(2).unwrap().expect("id 2 must resolve"); + assert_eq!(chunk.path, "b.md"); + + // New inserts start above the surviving range — no reuse. + let c_ids = store2 + .insert_chunks_with_ids(vec![drift_chunk("c.md", "content C0", 0)]) + .unwrap(); + assert_eq!(c_ids, vec![4]); + } } diff --git a/src/watch/mod.rs b/src/watch/mod.rs index 92abbcfa..42ddf268 100644 --- a/src/watch/mod.rs +++ b/src/watch/mod.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use ignore::gitignore::{Gitignore, GitignoreBuilder}; -use notify::{RecommendedWatcher, RecursiveMode, Watcher}; -use notify_debouncer_full::{new_debouncer, DebounceEventResult, Debouncer, FileIdMap}; +use notify::{RecommendedWatcher, RecursiveMode}; +use notify_debouncer_full::{new_debouncer, DebounceEventResult, Debouncer, RecommendedCache}; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::process::Command; @@ -67,7 +67,7 @@ pub enum FileEvent { /// 3. Batched events for efficient processing pub struct FileWatcher { root: PathBuf, - debouncer: Option>, + debouncer: Option>, receiver: Option>, /// Compiled .gitignore matcher for the repo root (None if no .gitignore found). gitignore: Option, @@ -195,17 +195,12 @@ impl FileWatcher { self.receiver = Some(rx); self.debouncer = Some(debouncer); - // Start watching the root directory + // Start watching the root directory (Debouncer implements Watcher directly + // since notify-debouncer-full 0.7 and tracks file-ID cache roots itself) if let Some(ref mut debouncer) = self.debouncer { debouncer - .watcher() .watch(&self.root, RecursiveMode::Recursive) .map_err(|e| anyhow!("Failed to watch directory: {}", e))?; - - // Also watch with the cache (for file ID tracking) - debouncer - .cache() - .add_root(&self.root, RecursiveMode::Recursive); } Ok(()) @@ -219,7 +214,7 @@ impl FileWatcher { /// Stop watching pub fn stop(&mut self) { if let Some(ref mut debouncer) = self.debouncer { - let _ = debouncer.watcher().unwatch(&self.root); + let _ = debouncer.unwatch(&self.root); } self.debouncer = None; self.receiver = None; diff --git a/tests/store_err_swallow_detector.rs b/tests/store_err_swallow_detector.rs new file mode 100644 index 00000000..e7270b5a --- /dev/null +++ b/tests/store_err_swallow_detector.rs @@ -0,0 +1,95 @@ +//! Guard against flattening a store `Err` into a missing hit on MCP +//! resolution paths ("search errors must not become empty results"). +//! +//! `VectorStore::get_chunk` returns `Result>`: `Ok(None)` is a +//! true miss ("this store does not hold that chunk"), `Err` is a broken +//! store. Collapsing the two makes a dead store render as an ordinary +//! empty or short result — the most misleading signal this system can emit. +//! This exact defect was fixed in sibling handlers and left in others four +//! times across review rounds (see AGENTS.md); review is the wrong +//! instrument, so this makes it a build failure. +//! +//! Two textual manifestations, both banned on a DIRECT `get_chunk` call: +//! +//! * **Rule A — `.ok()` on the call.** `store.get_chunk(id).ok()??` inside a +//! `filter_map` silently drops the hit. Binding first +//! (`let looked_up = store.get_chunk(id);` → note the `Err` → use +//! `looked_up.ok()`) is compliant — only the direct call is flagged. +//! * **Rule B — `if let Ok(Some(..)) = store.get_chunk(..)`.** The `Err` arm +//! vanishes into the else. Binding first and matching on the bound value +//! (after noting the `Err`) is compliant. +//! +//! Limitation (deliberate): line-based, like `caller_facing_literals.rs`. +//! A call wrapped across lines so the `.ok()` lands on the next line escapes +//! Rule A; the compliant population is written on one line and rustfmt keeps +//! it that way at these widths. + +use std::fs; +use std::path::{Path, PathBuf}; + +/// MCP handler sources: everything under `src/mcp/` except the sibling test +/// files, which construct fixtures and may name the APIs directly. +fn mcp_sources(repo_root: &Path) -> Vec { + let mut files = Vec::new(); + let dir = repo_root.join("src").join("mcp"); + let Ok(entries) = fs::read_dir(&dir) else { + panic!("cannot read {}", dir.display()); + }; + for entry in entries.flatten() { + let path = entry.path(); + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if path.extension().and_then(|e| e.to_str()) == Some("rs") + && !name.ends_with("_tests.rs") + && name != "tests.rs" + { + files.push(path); + } + } + files.sort(); + files +} + +fn repo_root() -> PathBuf { + // Integration tests run with CWD = repo root. + std::env::current_dir().expect("cwd") +} + +fn is_code(line: &str) -> bool { + let t = line.trim_start(); + !t.is_empty() && !t.starts_with("//") && !t.starts_with("///") +} + +#[test] +fn no_direct_get_chunk_flattened_on_mcp_paths() { + let mut violations: Vec = Vec::new(); + for file in mcp_sources(&repo_root()) { + let src = fs::read_to_string(&file).expect("readable"); + for (idx, line) in src.lines().enumerate() { + if !is_code(line) { + continue; + } + if line.contains(".get_chunk(") && line.contains(".ok()") { + violations.push(format!( + "{}:{}: direct get_chunk flattened with .ok(): {}", + file.display(), + idx + 1, + line.trim() + )); + } + if line.contains("if let Ok(Some(") && line.contains("= store.get_chunk(") { + violations.push(format!( + "{}:{}: get_chunk Err flattened in if-let scrutinee: {}", + file.display(), + idx + 1, + line.trim() + )); + } + } + } + assert!( + violations.is_empty(), + "store Err collapsed into a missing hit (bind the result, note the \ + Err via note_store_failure or propagate it with `?`):\n{}", + violations.join("\n") + ); +} diff --git a/tests/symbols_csharp_test.rs b/tests/symbols_csharp_test.rs index c455ee78..a92a895d 100644 --- a/tests/symbols_csharp_test.rs +++ b/tests/symbols_csharp_test.rs @@ -11,12 +11,12 @@ use std::path::PathBuf; use codesearch::constants::{SCIP_LMDB_DEFAULT_MAP_SIZE_MB, SCIP_LMDB_MAP_SIZE_MB_ENV}; use codesearch::symbols::csharp::CSharpSymbolIndexer; use codesearch::symbols::scip_parse; -use codesearch::symbols::{RebuildScope, SymbolIndexer}; +use codesearch::symbols::{ImpactQuery, KeyMatch, RebuildScope, SymbolIndexer}; use tempfile::TempDir; /// Sample JSON mimicking the output of scip-csharp for a small C# project. const SAMPLE_INDEX_JSON: &str = r#"{ - "metadata": {"version": "1.0", "tool_info": "scip-csharp"}, + "metadata": {"version": "2.0", "tool_info": "scip-csharp"}, "documents": [ { "relative_path": "src/Library/Calculator.cs", @@ -146,25 +146,25 @@ fn test_indexer_returns_empty_when_db_missing() { // open_scip_env creates the dir, so just verify it doesn't panic let _ = age; - // Test find_references with no data — should return Ok(empty) because - // resolve_canonical_key returns None when no LMDB tables exist. + // Test resolve_query with no data — should return NotFound because no + // LMDB tables exist. // // On CI runners with constrained resources, LMDB may fail to reopen after // the index_age call dropped its env (lock file not yet released). Accept - // both Ok(empty) and Err as valid outcomes — the important invariant is + // both NotFound and Err as valid outcomes — the important invariant is // that it never panics and never returns stale data. - let result = indexer.find_references(&db_path, "Calculator.Add"); + let result = indexer.resolve_query(&db_path, &ImpactQuery::Name("Calculator.Add".into())); match result { - Ok(refs) => assert!( - refs.is_empty(), - "Should return empty vec when no SCIP data exists, got {:?}", - refs + Ok(key) => assert_eq!( + key, + KeyMatch::NotFound, + "Should not resolve when no SCIP data exists, got {key:?}" ), Err(e) => { // LMDB reopen failed (e.g. lock contention on CI). This is // acceptable — the function correctly returns an error rather // than panicking or returning stale data. - eprintln!("Note: find_references returned Err (LMDB lock contention?): {e:#}"); + eprintln!("Note: resolve_query returned Err (LMDB lock contention?): {e:#}"); } } } @@ -172,7 +172,7 @@ fn test_indexer_returns_empty_when_db_missing() { #[test] fn test_parse_json_index_multiple_symbols_same_file() { let json = r#"{ - "metadata": {"version": "1.0", "tool_info": "test"}, + "metadata": {"version": "2.0", "tool_info": "test"}, "documents": [{ "relative_path": "src/A.cs", "occurrences": [ @@ -214,7 +214,7 @@ fn test_parse_json_index_multiple_symbols_same_file() { fn test_parse_json_index_role_fallback() { // When kind is empty string, should derive from symbol_roles let json = r#"{ - "metadata": {"version": "1.0", "tool_info": "test"}, + "metadata": {"version": "2.0", "tool_info": "test"}, "documents": [{ "relative_path": "src/A.cs", "occurrences": [ @@ -256,8 +256,8 @@ fn test_scip_lmdb_default_map_size_is_512mb() { /// /// We cannot directly inspect the `EnvOpenOptions` after the fact, so instead /// we exercise the observable behaviour: with a small custom map_size the -/// environment still opens successfully on an empty DB and `find_references` -/// returns `Ok(empty)` (no panic, no MDB_MAP_FULL). +/// environment still opens successfully on an empty DB and `resolve_query` +/// returns `Ok(NotFound)` (no panic, no MDB_MAP_FULL). /// /// A mutex serialises env-var mutation so this test is safe when `cargo test` /// runs suites in parallel. @@ -285,11 +285,11 @@ fn test_scip_lmdb_env_var_override() { let indexer = CSharpSymbolIndexer::new(); // On an empty DB the env-var path is exercised by open_scip_env(). - // The call must succeed and return an empty result set. - let result = indexer.find_references(&db_path, "SomeSymbol"); + // The call must succeed and report nothing found. + let result = indexer.resolve_query(&db_path, &ImpactQuery::Name("SomeSymbol".into())); assert!( - result.is_ok() && result.unwrap().is_empty(), - "Expected Ok(empty) from empty DB with env-var map_size override" + matches!(result, Ok(KeyMatch::NotFound)), + "Expected Ok(NotFound) from empty DB with env-var map_size override, got {result:?}" ); }); @@ -358,13 +358,18 @@ fn test_csharp_pipeline_smallsolution_roundtrip() { "No symbols indexed from fixture" ); - // Query: exact match for Calculator.Add should have >=2 occurrences + // Query: exact key for Calculator.Add should have >=2 occurrences + let add_key = "csharp SmallSolution.Library . Calculator#Add(int, int)."; + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::ExactKey(add_key.to_string())) + .expect("ExactKey resolution failed"); + let canonical = match resolved { + KeyMatch::Resolved(k) => k, + other => panic!("ExactKey must resolve verbatim, got {other:?}"), + }; let add_refs = indexer - .find_references( - db_path, - "csharp SmallSolution.Library . Calculator#Add(int, int).", - ) - .expect("find_references failed"); + .find_references_for_key(db_path, &canonical) + .expect("find_references_for_key failed"); assert!( add_refs.len() >= 2, "Expected >=2 refs for Calculator.Add, got {}", @@ -375,20 +380,75 @@ fn test_csharp_pipeline_smallsolution_roundtrip() { let defs: Vec<_> = add_refs.iter().filter(|r| r.kind == "definition").collect(); assert_eq!(defs.len(), 1, "Expected 1 definition for Calculator.Add"); - // Fuzzy query: "Add" should resolve to Calculator.Add + // Ambiguity contract: "Add" has two overloads — the adapter must list + // them, never pick one silently (the #238 contract: overloads come back + // as a sorted Ambiguous envelope; an explicit key selects one). + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::Name("Add".into())) + .expect("ambiguous-name resolution failed"); + let candidates = match resolved { + KeyMatch::Ambiguous(c) => c, + other => panic!("'Add' has two overloads and must come back Ambiguous, got {other:?}"), + }; + assert!( + candidates.len() >= 2, + "Expected >=2 Add overload candidates, got {candidates:?}" + ); + assert!( + candidates.iter().all(|k| k.contains("Calculator#Add")), + "Add candidates must be Calculator.Add overloads, got {candidates:?}" + ); + assert!( + candidates.windows(2).all(|w| w[0] <= w[1]), + "candidates must be sorted for deterministic output: {candidates:?}" + ); + let picked = candidates[0].clone(); + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::ExactKey(picked.clone())) + .expect("explicit selection failed"); + assert_eq!( + resolved, + KeyMatch::Resolved(picked.clone()), + "an explicit candidate selection must resolve to itself" + ); let fuzzy_refs = indexer - .find_references(db_path, "Add") - .expect("fuzzy find_references failed"); + .find_references_for_key(db_path, &picked) + .expect("find_references_for_key failed"); assert!( !fuzzy_refs.is_empty(), - "Fuzzy lookup for 'Add' should resolve to Calculator.Add" + "the picked Add overload must have references" + ); + + // A class name resolves to the class: methods register under their own + // simple name (see extract_simple_name), so "Calculator" is NOT + // ambiguous even though method keys contain the word. + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::Name("Calculator".into())) + .expect("class-name resolution failed"); + assert_eq!( + resolved, + KeyMatch::Resolved("csharp SmallSolution.Library . Calculator#".to_string()), + "class name must resolve to the class symbol" ); // Position-based lookup: find what's defined on Calculator.cs line 8 // Note: paths are solution-relative as produced by the helper + let resolved = indexer + .resolve_query( + db_path, + &ImpactQuery::Position { + file: PathBuf::from("Library/Calculator.cs"), + line: 8, + }, + ) + .expect("position resolution failed"); + let canonical = match resolved { + KeyMatch::Resolved(k) => k, + other => panic!("single-definition position must resolve, got {other:?}"), + }; let pos_refs = indexer - .find_references_by_position(db_path, &PathBuf::from("Library/Calculator.cs"), 8) - .expect("find_references_by_position failed"); + .find_references_for_key(db_path, &canonical) + .expect("find_references_for_key failed"); assert!( !pos_refs.is_empty(), "Position lookup for Library/Calculator.cs:8 should return references" diff --git a/tests/symbols_typescript_test.rs b/tests/symbols_typescript_test.rs index 3476c80e..bb463e25 100644 --- a/tests/symbols_typescript_test.rs +++ b/tests/symbols_typescript_test.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; use codesearch::symbols::typescript::TypeScriptSymbolIndexer; -use codesearch::symbols::{RebuildScope, SymbolIndexer}; +use codesearch::symbols::{ImpactQuery, KeyMatch, RebuildScope, SymbolIndexer}; use tempfile::TempDir; #[test] @@ -27,20 +27,20 @@ fn test_indexer_returns_empty_when_db_missing() { let age = indexer.index_age(&db_path); let _ = age; // open_scip_env creates the dir; just verify no panic. - // find_references with no data should return Ok(empty) because - // resolve_canonical_key returns None when no LMDB tables exist. - let result = indexer.find_references(&db_path, "add"); + // resolve_query with no data should return NotFound because no LMDB + // tables exist. + let result = indexer.resolve_query(&db_path, &ImpactQuery::Name("add".into())); match result { - Ok(refs) => assert!( - refs.is_empty(), - "Should return empty vec when no SCIP data exists, got {:?}", - refs + Ok(key) => assert_eq!( + key, + KeyMatch::NotFound, + "Should not resolve when no SCIP data exists, got {key:?}" ), Err(e) => { // LMDB reopen failed (e.g. lock contention on CI). Acceptable — // the important invariant is that it never panics or returns // stale data. - eprintln!("Note: find_references returned Err (LMDB lock contention?): {e:#}"); + eprintln!("Note: resolve_query returned Err (LMDB lock contention?): {e:#}"); } } } @@ -79,8 +79,8 @@ fn test_fixture_directory_shape() { /// Full pipeline integration test: scip-typescript subprocess → SCIP /// protobuf → LMDB → query, verifying that `find_impact`'s underlying -/// `find_references()` returns ALL call-sites of a TS symbol across -/// multiple files. +/// `resolve_query()` + `find_references_for_key()` return ALL call-sites +/// of a TS symbol across multiple files. /// /// Requires the `typescript_helper_integration` feature flag AND either: /// - `CODESEARCH_SCIP_TYPESCRIPT` env var pointing to a `scip-typescript` @@ -120,9 +120,21 @@ fn test_typescript_pipeline_ts_sample_roundtrip() { // Fuzzy lookup: "add" should resolve to the `add` function in math.ts // and return the definition plus all call-sites across both files // that import and call it. + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::Name("add".into())) + .expect("fuzzy resolution failed"); + let canonical = match resolved { + KeyMatch::Resolved(k) => k, + other => panic!("fuzzy 'add' should resolve uniquely, got {other:?}"), + }; let add_refs = indexer - .find_references(db_path, "add") - .expect("find_references failed"); + .find_references_for_key(db_path, &canonical) + .expect("find_references_for_key failed"); + // The answer must name the selected canonical identity. + assert!( + canonical.contains("add"), + "the resolved canonical key must identify `add`, got: {canonical}" + ); let defs: Vec<_> = add_refs.iter().filter(|r| r.kind == "definition").collect(); assert_eq!(defs.len(), 1, "Expected exactly 1 definition for `add`"); @@ -168,7 +180,7 @@ fn test_typescript_pipeline_ts_sample_roundtrip() { /// `CODESEARCH_TS_TEST_REAL` env var. Gated by the feature flag AND the env /// var, so it never runs in CI unless explicitly opted in. /// -/// Verifies: rebuild succeeds on a non-trivial codebase, `find_references` +/// Verifies: rebuild succeeds on a non-trivial codebase, `find_references_for_key` /// returns sensible multi-file results for a commonly-used symbol. #[test] #[cfg_attr(not(feature = "typescript_helper_integration"), ignore)] @@ -217,13 +229,24 @@ fn test_typescript_pipeline_real_project() { // `log` is a very commonly used symbol in the target project — expect // many call-sites across many files. for sym in &["log", "configureLogger"] { + let resolved = indexer + .resolve_query(db_path, &ImpactQuery::Name((*sym).into())) + .unwrap_or_else(|e| panic!("resolve_query({sym}) failed: {e:#}")); + let canonical = match resolved { + KeyMatch::Resolved(k) => k, + KeyMatch::Ambiguous(candidates) => { + eprintln!("resolve_query({sym:?}) is ambiguous: {candidates:?}"); + candidates[0].clone() + } + KeyMatch::NotFound => panic!("common symbol {sym} must resolve in a real project"), + }; let refs = indexer - .find_references(db_path, sym) - .unwrap_or_else(|e| panic!("find_references({sym}) failed: {e:#}")); + .find_references_for_key(db_path, &canonical) + .unwrap_or_else(|e| panic!("find_references_for_key({canonical}) failed: {e:#}")); let distinct_files: std::collections::HashSet<_> = refs.iter().map(|r| r.file.clone()).collect(); eprintln!( - "find_references({sym:?}): {} occurrences across {} files", + "find_references_for_key({canonical:?}): {} occurrences across {} files", refs.len(), distinct_files.len() ); @@ -234,10 +257,17 @@ fn test_typescript_pipeline_real_project() { ); } - // Negative test: unknown symbol returns empty, no panic. + // Negative test: unknown symbol resolves NotFound, no panic. let unknown = indexer - .find_references(db_path, "thisSymbolDoesNotExist_xyzzy_12345") - .expect("find_references on unknown symbol should not error"); - assert!(unknown.is_empty(), "Unknown symbol should return empty"); - eprintln!("negative test OK: unknown symbol returned 0 results"); + .resolve_query( + db_path, + &ImpactQuery::Name("thisSymbolDoesNotExist_xyzzy_12345".into()), + ) + .expect("resolve_query on unknown symbol should not error"); + assert_eq!( + unknown, + KeyMatch::NotFound, + "Unknown symbol should resolve NotFound" + ); + eprintln!("negative test OK: unknown symbol reported NotFound"); }