From d1333b248c18354ac51b9cca8c3012bf65c1744e Mon Sep 17 00:00:00 2001 From: arreyder Date: Tue, 9 Jun 2026 23:49:21 -0500 Subject: [PATCH 1/9] hybrid semantic search: embeddings + KNN + RRF fusion (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds dense-vector semantic search blended with the existing lexical ranking, so conceptually-related memories surface even with no shared terms. - internal/embed: pluggable Embedder; Ollama /api/embeddings impl; FromEnv (EMBED_URL/EMBED_MODEL/EMBED_DIM). Disabled no-op when unconfigured → lexical-only, never fails. - schema: knn_vector_768 DenseVectorField (cosine) + `embedding` field (stored=false so vectors never bloat responses; indexed for KNN). - store/bulk_store: embed title+content on write. update: re-embed only when content/title change (tag/importance updates skip it), fetching the missing half so the vector reflects both. - search: when enabled, embed the query, KNN alongside lexical edismax, fuse with reciprocal rank fusion (k=60). Over-fetch fusionK then trim to limit. semantic=false opts out; start>0 (pagination) forces lexical-only. Any embed/KNN error degrades to lexical-only. - client.KNNQuery (POSTs the vector to dodge URL-length limits) + formatVector. - cmd/solr-mem-backfill: one-shot re-embed of existing memories (idempotent). Tests: embedder (request/parse/dim, disabled), fuseResponses (RRF order, dedup, semantic-only inclusion, limit, nil). build/vet/test/gofmt clean. Deploy: schema reload (docker cp + cores RELOAD) + EMBED_* env on the server + run backfill. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/solr-mem-backfill/main.go | 88 ++++++++++++++++++ cmd/solr-mem-server/bulk_store_tool.go | 1 + cmd/solr-mem-server/embedding.go | 47 ++++++++++ cmd/solr-mem-server/fuse.go | 79 ++++++++++++++++ cmd/solr-mem-server/fuse_test.go | 61 ++++++++++++ cmd/solr-mem-server/main.go | 12 +++ cmd/solr-mem-server/search_tool.go | 37 +++++++- cmd/solr-mem-server/store_tool.go | 1 + cmd/solr-mem-server/tools.go | 3 + cmd/solr-mem-server/update_tool.go | 20 ++++ internal/embed/embed.go | 123 +++++++++++++++++++++++++ internal/embed/embed_test.go | 59 ++++++++++++ internal/solr/client.go | 49 ++++++++++ internal/solr/types.go | 3 + solr/managed-schema.xml | 8 ++ 15 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 cmd/solr-mem-backfill/main.go create mode 100644 cmd/solr-mem-server/embedding.go create mode 100644 cmd/solr-mem-server/fuse.go create mode 100644 cmd/solr-mem-server/fuse_test.go create mode 100644 internal/embed/embed.go create mode 100644 internal/embed/embed_test.go diff --git a/cmd/solr-mem-backfill/main.go b/cmd/solr-mem-backfill/main.go new file mode 100644 index 0000000..b396b75 --- /dev/null +++ b/cmd/solr-mem-backfill/main.go @@ -0,0 +1,88 @@ +// Command solr-mem-backfill embeds existing memories so semantic search can +// find them. It re-embeds every memory (idempotent) — title + content via the +// configured embedder — and atomically sets the `embedding` field. +// +// Env: SOLR_URL (default http://localhost:8983/solr/memories), EMBED_URL, +// EMBED_MODEL, EMBED_DIM (same as the server). +package main + +import ( + "context" + "flag" + "log" + "os" + "strings" + "time" + + "github.com/arreyder/solr-mem/internal/embed" + "github.com/arreyder/solr-mem/internal/solr" +) + +func main() { + batch := flag.Int("batch", 100, "docs per page / update batch") + flag.Parse() + + solrURL := os.Getenv("SOLR_URL") + if solrURL == "" { + solrURL = "http://localhost:8983/solr/memories" + } + client := solr.NewClient(solrURL) + + emb := embed.FromEnv() + if !emb.Enabled() { + log.Fatal("EMBED_URL not set — nothing to backfill (embeddings disabled)") + } + + ctx := context.Background() + start := 0 + total, embedded, failed := 0, 0, 0 + + for { + resp, err := client.Query(ctx, solr.QueryParams{ + Query: "*:*", + Rows: *batch, + Start: start, + Sort: "id asc", // stable paging + Fields: []string{"id", "title", "content"}, + Highlight: false, + }) + if err != nil { + log.Fatalf("query at start=%d: %v", start, err) + } + if len(resp.Docs) == 0 { + break + } + + var updates []map[string]any + for _, d := range resp.Docs { + total++ + id, _ := d["id"].(string) + title, _ := d["title"].(string) + content, _ := d["content"].(string) + text := strings.TrimSpace(title + "\n\n" + content) + if id == "" || text == "" { + continue + } + vec, err := emb.Embed(ctx, text) + if err != nil || len(vec) == 0 { + log.Printf("embed failed id=%s: %v", id, err) + failed++ + continue + } + updates = append(updates, map[string]any{ + "id": id, + "embedding": map[string]any{"set": vec}, + }) + embedded++ + } + + if err := client.BulkUpdate(ctx, updates); err != nil { + log.Fatalf("bulk update at start=%d: %v", start, err) + } + log.Printf("progress: %d seen, %d embedded, %d failed", total, embedded, failed) + start += *batch + time.Sleep(50 * time.Millisecond) // be gentle on the embed service + } + + log.Printf("DONE: %d memories, %d embedded, %d failed", total, embedded, failed) +} diff --git a/cmd/solr-mem-server/bulk_store_tool.go b/cmd/solr-mem-server/bulk_store_tool.go index 6f64a75..bfadadd 100644 --- a/cmd/solr-mem-server/bulk_store_tool.go +++ b/cmd/solr-mem-server/bulk_store_tool.go @@ -78,6 +78,7 @@ func bulkStoreMemoriesTool(ctx context.Context, args map[string]any) (any, error SessionID: getString(m, "session_id"), RelatedIDs: getStringSlice(m, "related_ids"), Format: format, + Embedding: embedMemoryText(ctx, scrubbedTitle, scrubbedContent), }) } diff --git a/cmd/solr-mem-server/embedding.go b/cmd/solr-mem-server/embedding.go new file mode 100644 index 0000000..4e4f95d --- /dev/null +++ b/cmd/solr-mem-server/embedding.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/arreyder/solr-mem/internal/solr" +) + +// embedMemoryText embeds a memory's semantic text (title + content). Returns +// nil when embeddings are disabled or on error, so store/update degrade to +// lexical-only instead of failing the write. +func embedMemoryText(ctx context.Context, title, content string) []float32 { + if !embedder.Enabled() { + return nil + } + text := strings.TrimSpace(title + "\n\n" + content) + if text == "" { + return nil + } + vec, err := embedder.Embed(ctx, text) + if err != nil { + log.Printf("embedding failed (proceeding without vector): %v", err) + return nil + } + return vec +} + +// currentTitleContent fetches a memory's stored title and content. Used when +// re-embedding on update where only one of the two was supplied, so the vector +// still reflects both fields. +func currentTitleContent(ctx context.Context, id string) (title, content string) { + resp, err := solrClient.Query(ctx, solr.QueryParams{ + Query: fmt.Sprintf("id:%q", id), + Rows: 1, + Fields: []string{"title", "content"}, + Highlight: false, + }) + if err != nil || resp == nil || len(resp.Docs) == 0 { + return "", "" + } + title, _ = resp.Docs[0]["title"].(string) + content, _ = resp.Docs[0]["content"].(string) + return title, content +} diff --git a/cmd/solr-mem-server/fuse.go b/cmd/solr-mem-server/fuse.go new file mode 100644 index 0000000..1d56711 --- /dev/null +++ b/cmd/solr-mem-server/fuse.go @@ -0,0 +1,79 @@ +package main + +import ( + "sort" + + "github.com/arreyder/solr-mem/internal/solr" +) + +// rrfK is the reciprocal-rank-fusion constant. 60 is the widely-used default +// from the original RRF paper; it damps the influence of any single ranker's +// top positions so the two lists combine smoothly. +const rrfK = 60 + +func docID(d map[string]any) string { + s, _ := d["id"].(string) + return s +} + +// fuseResponses combines lexical and semantic (KNN) result lists with +// reciprocal rank fusion: score(d) = Σ 1/(rrfK + rank) across the lists it +// appears in. Returns a response with docs ordered by fused score (desc), +// de-duplicated by id and capped to limit, with highlighting merged from both. +// Ties break by lexical order first, then semantic — deterministic regardless +// of map iteration. +func fuseResponses(lexical, semantic *solr.QueryResponse, limit int) *solr.QueryResponse { + scores := map[string]float64{} + docByID := map[string]map[string]any{} + var order []string // deterministic seed order: lexical first, then semantic-only + seen := map[string]bool{} + + accumulate := func(resp *solr.QueryResponse) { + if resp == nil { + return + } + for rank, d := range resp.Docs { + id := docID(d) + if id == "" { + continue + } + scores[id] += 1.0 / float64(rrfK+rank+1) // rank is 0-based + if !seen[id] { + seen[id] = true + order = append(order, id) + docByID[id] = d + } + } + } + accumulate(lexical) + accumulate(semantic) + + sort.SliceStable(order, func(i, j int) bool { + return scores[order[i]] > scores[order[j]] + }) + if limit > 0 && len(order) > limit { + order = order[:limit] + } + + docs := make([]map[string]any, 0, len(order)) + for _, id := range order { + docs = append(docs, docByID[id]) + } + + hl := map[string]map[string][]string{} + for _, resp := range []*solr.QueryResponse{lexical, semantic} { + if resp == nil { + continue + } + for k, v := range resp.Highlighting { + hl[k] = v + } + } + + out := &solr.QueryResponse{Docs: docs, Highlighting: hl} + if lexical != nil { + out.NumFound = lexical.NumFound + out.Facets = lexical.Facets + } + return out +} diff --git a/cmd/solr-mem-server/fuse_test.go b/cmd/solr-mem-server/fuse_test.go new file mode 100644 index 0000000..e6cf9ca --- /dev/null +++ b/cmd/solr-mem-server/fuse_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" + + "github.com/arreyder/solr-mem/internal/solr" +) + +func resp(ids ...string) *solr.QueryResponse { + docs := make([]map[string]any, len(ids)) + for i, id := range ids { + docs[i] = map[string]any{"id": id} + } + return &solr.QueryResponse{NumFound: len(ids), Docs: docs} +} + +func order(r *solr.QueryResponse) []string { + out := make([]string, len(r.Docs)) + for i, d := range r.Docs { + out[i] = docID(d) + } + return out +} + +func TestFuseResponses_RRF(t *testing.T) { + // b ranks high in both lists -> should win. d is semantic-only -> still included. + lexical := resp("a", "b", "c") + semantic := resp("b", "d", "a") + + fused := fuseResponses(lexical, semantic, 10) + got := order(fused) + + // b: 1/61 + 1/61 (rank0 both) = highest. + if got[0] != "b" { + t.Fatalf("expected 'b' first, got %v", got) + } + // All four unique ids present, deduped. + if len(got) != 4 { + t.Fatalf("expected 4 unique docs, got %v", got) + } + // d (semantic-only) is included. + if !contains(got, "d") { + t.Errorf("semantic-only 'd' missing: %v", got) + } +} + +func TestFuseResponses_LimitAndNilSemantic(t *testing.T) { + fused := fuseResponses(resp("a", "b", "c"), nil, 2) + if got := order(fused); len(got) != 2 || got[0] != "a" { + t.Fatalf("limit/nil-semantic: got %v", got) + } +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/cmd/solr-mem-server/main.go b/cmd/solr-mem-server/main.go index 3d01776..5f2b2c7 100644 --- a/cmd/solr-mem-server/main.go +++ b/cmd/solr-mem-server/main.go @@ -8,6 +8,7 @@ import ( "net/http" "os" + "github.com/arreyder/solr-mem/internal/embed" "github.com/arreyder/solr-mem/internal/solr" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -15,6 +16,10 @@ import ( var solrClient *solr.Client var codeClient *solr.Client +// embedder produces query/document embeddings for semantic search. Disabled +// (lexical-only) unless EMBED_URL is configured. +var embedder embed.Embedder = embed.Disabled{} + // indexerControlURL is the base URL of the indexer's force-reindex control // endpoint. Defaults to the co-located indexer on localhost. var indexerControlURL = envOrDefault("INDEXER_CONTROL_URL", "http://127.0.0.1:7071") @@ -39,6 +44,13 @@ func main() { } codeClient = solr.NewClient(codeURL) + embedder = embed.FromEnv() + if embedder.Enabled() { + log.Printf("Semantic search enabled: embeddings via %s (dim %d)", os.Getenv("EMBED_URL"), embedder.Dim()) + } else { + log.Printf("Semantic search disabled (EMBED_URL unset); lexical-only") + } + // Start expiration sweeper ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/cmd/solr-mem-server/search_tool.go b/cmd/solr-mem-server/search_tool.go index 8aa70aa..7fa6cd3 100644 --- a/cmd/solr-mem-server/search_tool.go +++ b/cmd/solr-mem-server/search_tool.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log" "strings" "github.com/arreyder/solr-mem/internal/solr" @@ -15,9 +16,22 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { return nil, fmt.Errorf("query is required") } + limit := getInt(args, "limit", 10) + + // Hybrid semantic search: blend lexical ranking with vector KNN. On when an + // embedder is configured and the caller hasn't opted out. We over-fetch + // (fusionK) from each ranker so fusion has enough candidates, then trim to + // limit at the end. Pagination (start) forces lexical-only — KNN+RRF has no + // stable global offset. + semantic := embedder.Enabled() && getBool(args, "semantic", true) && getInt(args, "start", 0) == 0 + fusionK := limit + if semantic && fusionK < 50 { + fusionK = 50 + } + params := solr.QueryParams{ Query: query, - Rows: getInt(args, "limit", 10), + Rows: fusionK, Start: getInt(args, "start", 0), Highlight: getBool(args, "highlight", true), Facet: getBool(args, "facet", false), @@ -83,6 +97,22 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { return nil, fmt.Errorf("search failed: %w", err) } + // Blend in semantic (KNN) ranking via reciprocal rank fusion. Degrades to + // lexical-only on any embed/KNN error so search never hard-fails on the + // optional path. + if semantic { + if vec, eerr := embedder.Embed(ctx, query); eerr != nil { + log.Printf("query embedding failed (lexical-only): %v", eerr) + } else if len(vec) > 0 { + knn, kerr := solrClient.KNNQuery(ctx, "embedding", vec, fusionK, params.FilterQueries, params.Fields) + if kerr != nil { + log.Printf("knn search failed (lexical-only): %v", kerr) + } else { + resp = fuseResponses(resp, knn, fusionK) + } + } + } + // Cap per session so one chatty session can't dominate results. // Default 3; pass 0 to disable. sessionCap := getInt(args, "session_cap", 3) @@ -93,6 +123,11 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { }, sessionCap) } + // Trim to the requested limit (we over-fetched fusionK for fusion headroom). + if limit > 0 && len(resp.Docs) > limit { + resp.Docs = resp.Docs[:limit] + } + // Credit a retrieval for the memories actually surfaced (fire-and-forget). // Pass track:false for maintenance/bulk scans so they don't inflate the signal. recordRetrievalsAsync(resp.Docs, getBool(args, "track", true)) diff --git a/cmd/solr-mem-server/store_tool.go b/cmd/solr-mem-server/store_tool.go index c17459c..2054ea3 100644 --- a/cmd/solr-mem-server/store_tool.go +++ b/cmd/solr-mem-server/store_tool.go @@ -50,6 +50,7 @@ func storeMemoryTool(ctx context.Context, args map[string]any) (any, error) { RelatedIDs: getStringSlice(args, "related_ids"), Format: format, } + doc.Embedding = embedMemoryText(ctx, scrubbedTitle, scrubbedContent) if err := solrClient.Add(ctx, doc); err != nil { return nil, fmt.Errorf("failed to store memory: %w", err) diff --git a/cmd/solr-mem-server/tools.go b/cmd/solr-mem-server/tools.go index 0a47e8b..c1857e6 100644 --- a/cmd/solr-mem-server/tools.go +++ b/cmd/solr-mem-server/tools.go @@ -56,6 +56,8 @@ func ToolSchemas() []ToolDefinition { **When to use**: Find relevant memories by content, tags, type, agent, or time range. Uses edismax with field boosting (content^3, title^2, tags^1.5) and recency boost. +**Semantic (hybrid) search**: when embeddings are enabled, results blend lexical (BM25) ranking with vector similarity (KNN) via reciprocal rank fusion — so conceptually-related memories surface even with no shared words. On by default; pass semantic=false for pure lexical (exact/debugging). Ignored when start>0 (pagination is lexical-only). + **Match modes**: By default a query requires most terms to match (mm 75%), so a long OR-style list of synonyms can return nothing. Pass match="any" for OR-style recall (any term matches) — best when throwing several candidate terms at the store; match="all" requires every term. **Lean payloads**: Pass fields (e.g. ["id","title","importance","memory_type"]) to project only those fields and avoid returning full content bodies. Use start for pagination (offset). @@ -66,6 +68,7 @@ func ToolSchemas() []ToolDefinition { **Optional**: match, fields, start, agent_id, memory_type, tags, source, importance_min, from, to, limit, highlight, facet, session_id, lifetime, session_cap, track`, InputSchema: NewObjectSchema(map[string]any{ "query": prop("string", "Full-text search query (required)"), + "semantic": prop("boolean", "Hybrid semantic+lexical search (default true when embeddings enabled). Set false for pure lexical/exact matching."), "match": prop("string", "Match mode: 'most' (default, mm 75%), 'any' (OR — any term), 'all' (every term). Use 'any' for synonym/OR-style recall."), "track": prop("boolean", "Record a retrieval for surfaced memories (default true). Set false for maintenance/bulk scans so they don't inflate usage stats."), "fields": arrayPropSchema(prop("string", "Field name"), "Project only these fields (Solr fl) for lean payloads; id is always included"), diff --git a/cmd/solr-mem-server/update_tool.go b/cmd/solr-mem-server/update_tool.go index 26d25af..929fc34 100644 --- a/cmd/solr-mem-server/update_tool.go +++ b/cmd/solr-mem-server/update_tool.go @@ -75,6 +75,26 @@ func updateMemoryTool(ctx context.Context, args map[string]any) (any, error) { return nil, fmt.Errorf("no fields to update") } + // Re-embed only when the semantic text (content/title) changed. Tag / + // importance / related_ids updates (e.g. the sleep-pass) skip this. If only + // one of the two changed, fetch the other so the vector reflects both. + newContent, contentChanged := fields["content"].(string) + newTitle, titleChanged := fields["title"].(string) + if embedder.Enabled() && (contentChanged || titleChanged) { + if !contentChanged || !titleChanged { + curTitle, curContent := currentTitleContent(ctx, id) + if !titleChanged { + newTitle = curTitle + } + if !contentChanged { + newContent = curContent + } + } + if vec := embedMemoryText(ctx, newTitle, newContent); vec != nil { + fields["embedding"] = vec + } + } + fields["updated_at"] = time.Now().UTC().Format(time.RFC3339) if err := solrClient.Update(ctx, id, fields); err != nil { diff --git a/internal/embed/embed.go b/internal/embed/embed.go new file mode 100644 index 0000000..8b40377 --- /dev/null +++ b/internal/embed/embed.go @@ -0,0 +1,123 @@ +// Package embed provides text embeddings for semantic memory search. +// +// The default backend is an Ollama-compatible HTTP endpoint. When no endpoint +// is configured the package returns a disabled embedder, so the rest of the +// system degrades gracefully to lexical-only search instead of failing. +package embed + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" +) + +// Embedder turns text into a dense vector. Implementations must be safe for +// concurrent use. +type Embedder interface { + // Enabled reports whether embeddings are available. When false, callers + // should fall back to lexical-only behavior. + Enabled() bool + // Dim is the vector dimension (must match the Solr DenseVectorField). + Dim() int + // Embed returns the embedding for text. + Embed(ctx context.Context, text string) ([]float32, error) +} + +// FromEnv builds an Embedder from EMBED_URL / EMBED_MODEL / EMBED_DIM. +// If EMBED_URL is empty, returns a disabled embedder (semantic search off). +func FromEnv() Embedder { + url := os.Getenv("EMBED_URL") + if url == "" { + return Disabled{} + } + model := os.Getenv("EMBED_MODEL") + if model == "" { + model = "nomic-embed-text" + } + dim := 768 + if v := os.Getenv("EMBED_DIM"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + dim = n + } + } + return NewOllama(url, model, dim) +} + +// Disabled is a no-op embedder used when no backend is configured. +type Disabled struct{} + +func (Disabled) Enabled() bool { return false } +func (Disabled) Dim() int { return 0 } +func (Disabled) Embed(context.Context, string) ([]float32, error) { + return nil, nil +} + +// Ollama embeds via an Ollama-compatible /api/embeddings endpoint. +type Ollama struct { + baseURL string + model string + dim int + client *http.Client +} + +// NewOllama builds an Ollama embedder. baseURL is the host root, e.g. +// "http://pax99.local:11434". +func NewOllama(baseURL, model string, dim int) *Ollama { + return &Ollama{ + baseURL: baseURL, + model: model, + dim: dim, + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (o *Ollama) Enabled() bool { return true } +func (o *Ollama) Dim() int { return o.dim } + +func (o *Ollama) Embed(ctx context.Context, text string) ([]float32, error) { + body, err := json.Marshal(map[string]any{"model": o.model, "prompt": text}) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + o.baseURL+"/api/embeddings", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := o.client.Do(req) + if err != nil { + return nil, fmt.Errorf("embed request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("embed returned %d: %s", resp.StatusCode, b) + } + return parseEmbedding(resp.Body, o.dim) +} + +// parseEmbedding decodes an Ollama embeddings response and validates the +// dimension. Split out for testing. +func parseEmbedding(r io.Reader, wantDim int) ([]float32, error) { + var out struct { + Embedding []float32 `json:"embedding"` + } + if err := json.NewDecoder(r).Decode(&out); err != nil { + return nil, fmt.Errorf("decode embedding: %w", err) + } + if len(out.Embedding) == 0 { + return nil, fmt.Errorf("embedding response had no vector") + } + if wantDim > 0 && len(out.Embedding) != wantDim { + return nil, fmt.Errorf("embedding dim %d != expected %d", len(out.Embedding), wantDim) + } + return out.Embedding, nil +} diff --git a/internal/embed/embed_test.go b/internal/embed/embed_test.go new file mode 100644 index 0000000..6c48563 --- /dev/null +++ b/internal/embed/embed_test.go @@ -0,0 +1,59 @@ +package embed + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestDisabled(t *testing.T) { + var e Embedder = Disabled{} + if e.Enabled() { + t.Fatal("Disabled must report not enabled") + } + v, err := e.Embed(context.Background(), "x") + if err != nil || v != nil { + t.Fatalf("Disabled.Embed = %v, %v; want nil, nil", v, err) + } +} + +func TestOllamaEmbed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/embeddings" { + t.Errorf("path = %q", r.URL.Path) + } + var req map[string]any + b, _ := io.ReadAll(r.Body) + json.Unmarshal(b, &req) + if req["model"] != "nomic-embed-text" || req["prompt"] != "hello" { + t.Errorf("unexpected request body: %v", req) + } + io.WriteString(w, `{"embedding":[0.1,0.2,0.3]}`) + })) + defer srv.Close() + + o := NewOllama(srv.URL, "nomic-embed-text", 3) + if !o.Enabled() || o.Dim() != 3 { + t.Fatalf("Enabled/Dim wrong: %v %d", o.Enabled(), o.Dim()) + } + v, err := o.Embed(context.Background(), "hello") + if err != nil { + t.Fatalf("Embed err: %v", err) + } + if len(v) != 3 || v[0] != 0.1 { + t.Fatalf("vector = %v", v) + } +} + +func TestParseEmbeddingDimMismatch(t *testing.T) { + if _, err := parseEmbedding(strings.NewReader(`{"embedding":[1,2]}`), 3); err == nil { + t.Fatal("expected dim-mismatch error") + } + if _, err := parseEmbedding(strings.NewReader(`{"embedding":[]}`), 0); err == nil { + t.Fatal("expected empty-vector error") + } +} diff --git a/internal/solr/client.go b/internal/solr/client.go index 893a282..d89434e 100644 --- a/internal/solr/client.go +++ b/internal/solr/client.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "time" ) @@ -311,6 +312,54 @@ func (c *Client) MoreLikeThis(ctx context.Context, id string, rows int, filterQu return ParseQueryResponse(resp.Body) } +// KNNQuery runs an approximate-nearest-neighbor search over a dense vector +// field. The query is POSTed (not GET) because the vector text can exceed URL +// length limits. Returns docs in similarity order. filterQueries are applied as +// pre-filters; fields projects the returned docs. +func (c *Client) KNNQuery(ctx context.Context, field string, vec []float32, topK int, filterQueries, fields []string) (*QueryResponse, error) { + form := url.Values{} + form.Set("q", fmt.Sprintf("{!knn f=%s topK=%d}%s", field, topK, formatVector(vec))) + form.Set("rows", strconv.Itoa(topK)) + form.Set("wt", "json") + if len(fields) > 0 { + form.Set("fl", strings.Join(fields, ",")) + } + for _, fq := range filterQueries { + form.Add("fq", fq) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/select", strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("solr knn: %w", err) + } + defer resp.Body.Close() + if err := c.checkResponse(resp); err != nil { + return nil, err + } + return ParseQueryResponse(resp.Body) +} + +// formatVector renders a float vector as Solr's "[a,b,c]" literal. +func formatVector(vec []float32) string { + var b strings.Builder + b.WriteByte('[') + for i, f := range vec { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(strconv.FormatFloat(float64(f), 'g', -1, 32)) + } + b.WriteByte(']') + return b.String() +} + // DeleteByQuery removes all documents matching the given query. func (c *Client) DeleteByQuery(ctx context.Context, query string) error { payload := map[string]any{ diff --git a/internal/solr/types.go b/internal/solr/types.go index d22ff70..e6a003a 100644 --- a/internal/solr/types.go +++ b/internal/solr/types.go @@ -20,6 +20,9 @@ type Document struct { SessionID string `json:"session_id,omitempty"` RelatedIDs []string `json:"related_ids,omitempty"` Format string `json:"format,omitempty"` + // Embedding is the dense semantic vector. Omitted when embeddings are + // disabled so the field never reaches Solr in lexical-only mode. + Embedding []float32 `json:"embedding,omitempty"` } // QueryParams holds parameters for a Solr search query. diff --git a/solr/managed-schema.xml b/solr/managed-schema.xml index 39f9cd4..57cbd2e 100644 --- a/solr/managed-schema.xml +++ b/solr/managed-schema.xml @@ -9,6 +9,10 @@ + + + @@ -54,4 +58,8 @@ + + + From ce9c731311bb7537dd78d3dc560bd81b4f0c2d13 Mon Sep 17 00:00:00 2001 From: arreyder Date: Tue, 9 Jun 2026 23:51:53 -0500 Subject: [PATCH 2/9] embed: truncate input to model context limit (EMBED_MAX_CHARS, default 6000) 16/596 memories failed backfill with 'input length exceeds context length' (nomic-embed-text ~2048 tokens). Truncate embed input by runes; title+head carries the semantic signal. Configurable via EMBED_MAX_CHARS. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/embed/embed.go | 43 ++++++++++++++++++++++++++++-------- internal/embed/embed_test.go | 16 ++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/internal/embed/embed.go b/internal/embed/embed.go index 8b40377..f6abe93 100644 --- a/internal/embed/embed.go +++ b/internal/embed/embed.go @@ -46,7 +46,13 @@ func FromEnv() Embedder { dim = n } } - return NewOllama(url, model, dim) + o := NewOllama(url, model, dim) + if v := os.Getenv("EMBED_MAX_CHARS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + o.maxChars = n + } + } + return o } // Disabled is a no-op embedder used when no backend is configured. @@ -60,20 +66,26 @@ func (Disabled) Embed(context.Context, string) ([]float32, error) { // Ollama embeds via an Ollama-compatible /api/embeddings endpoint. type Ollama struct { - baseURL string - model string - dim int - client *http.Client + baseURL string + model string + dim int + maxChars int // truncate input to this many runes (model context guard) + client *http.Client } +// defaultMaxChars keeps embed input under typical small-model context windows +// (nomic-embed-text ~2048 tokens). Conservative at ~4 chars/token. +const defaultMaxChars = 6000 + // NewOllama builds an Ollama embedder. baseURL is the host root, e.g. // "http://pax99.local:11434". func NewOllama(baseURL, model string, dim int) *Ollama { return &Ollama{ - baseURL: baseURL, - model: model, - dim: dim, - client: &http.Client{Timeout: 30 * time.Second}, + baseURL: baseURL, + model: model, + dim: dim, + maxChars: defaultMaxChars, + client: &http.Client{Timeout: 30 * time.Second}, } } @@ -81,6 +93,7 @@ func (o *Ollama) Enabled() bool { return true } func (o *Ollama) Dim() int { return o.dim } func (o *Ollama) Embed(ctx context.Context, text string) ([]float32, error) { + text = truncateRunes(text, o.maxChars) body, err := json.Marshal(map[string]any{"model": o.model, "prompt": text}) if err != nil { return nil, err @@ -104,6 +117,18 @@ func (o *Ollama) Embed(ctx context.Context, text string) ([]float32, error) { return parseEmbedding(resp.Body, o.dim) } +// truncateRunes caps s to at most max runes (UTF-8 safe). max<=0 means no cap. +func truncateRunes(s string, max int) string { + if max <= 0 { + return s + } + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) +} + // parseEmbedding decodes an Ollama embeddings response and validates the // dimension. Split out for testing. func parseEmbedding(r io.Reader, wantDim int) ([]float32, error) { diff --git a/internal/embed/embed_test.go b/internal/embed/embed_test.go index 6c48563..38f8f89 100644 --- a/internal/embed/embed_test.go +++ b/internal/embed/embed_test.go @@ -49,6 +49,22 @@ func TestOllamaEmbed(t *testing.T) { } } +func TestTruncateRunes(t *testing.T) { + if got := truncateRunes("hello", 3); got != "hel" { + t.Errorf("got %q", got) + } + if got := truncateRunes("hi", 10); got != "hi" { + t.Errorf("no-trunc got %q", got) + } + if got := truncateRunes("hello", 0); got != "hello" { + t.Errorf("max<=0 must be no-op, got %q", got) + } + // multi-byte safe + if got := truncateRunes("héllo", 2); got != "hé" { + t.Errorf("utf8 got %q", got) + } +} + func TestParseEmbeddingDimMismatch(t *testing.T) { if _, err := parseEmbedding(strings.NewReader(`{"embedding":[1,2]}`), 3); err == nil { t.Fatal("expected dim-mismatch error") From 8dffd4c151d931ad9fce9c7b4a59a1cffab35cdc Mon Sep 17 00:00:00 2001 From: arreyder Date: Wed, 10 Jun 2026 00:06:04 -0500 Subject: [PATCH 3/9] knn: force defType=lucene so {!knn} isn't parsed as edismax text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /select handler defaults to edismax, which doesn't honor the leading {!knn} parser switch — it tokenized the 768-float vector literal across qf fields, exceeding maxClauseCount (1024) -> 500. Send defType=lucene (+ drop facet/hl) so the knn parser is used. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/solr/client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/solr/client.go b/internal/solr/client.go index d89434e..a98aab5 100644 --- a/internal/solr/client.go +++ b/internal/solr/client.go @@ -319,6 +319,13 @@ func (c *Client) MoreLikeThis(ctx context.Context, id string, rows int, filterQu func (c *Client) KNNQuery(ctx context.Context, field string, vec []float32, topK int, filterQueries, fields []string) (*QueryResponse, error) { form := url.Values{} form.Set("q", fmt.Sprintf("{!knn f=%s topK=%d}%s", field, topK, formatVector(vec))) + // Force the lucene parser: the /select handler defaults to edismax, which + // does NOT honor the leading {!knn} parser switch and would tokenize the + // vector literal as text (blowing maxClauseCount). Also drop the handler's + // facet/highlight/recency-boost defaults — irrelevant to a KNN sub-query. + form.Set("defType", "lucene") + form.Set("facet", "false") + form.Set("hl", "off") form.Set("rows", strconv.Itoa(topK)) form.Set("wt", "json") if len(fields) > 0 { From e5e51730a8081b4fdfd24cf3b20ed53d7a2c550c Mon Sep 17 00:00:00 2001 From: arreyder Date: Wed, 10 Jun 2026 00:07:01 -0500 Subject: [PATCH 4/9] fuse: NumFound reflects returned docs (semantic-only hits aren't in lexical count) Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/solr-mem-server/fuse.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/solr-mem-server/fuse.go b/cmd/solr-mem-server/fuse.go index 1d56711..04c9793 100644 --- a/cmd/solr-mem-server/fuse.go +++ b/cmd/solr-mem-server/fuse.go @@ -72,8 +72,13 @@ func fuseResponses(lexical, semantic *solr.QueryResponse, limit int) *solr.Query out := &solr.QueryResponse{Docs: docs, Highlighting: hl} if lexical != nil { - out.NumFound = lexical.NumFound out.Facets = lexical.Facets + out.NumFound = lexical.NumFound + } + // Don't report fewer than we're actually returning — semantic-only hits + // aren't counted in the lexical NumFound. + if len(docs) > out.NumFound { + out.NumFound = len(docs) } return out } From 5294b3a8cea6db3c9471b87a6fda9f84bdd2fd1e Mon Sep 17 00:00:00 2001 From: arreyder Date: Wed, 10 Jun 2026 00:12:37 -0500 Subject: [PATCH 5/9] embed: nomic task prefixes (search_document/search_query) for asymmetric retrieval nomic-embed-text is trained with task prefixes; without them query/doc vectors are misaligned and precision suffers. Split Embed into EmbedDocument/EmbedQuery; prefix stored text with 'search_document: ' and queries with 'search_query: ' (auto for nomic models, overridable via EMBED_DOC_PREFIX/EMBED_QUERY_PREFIX). Requires re-backfill so stored vectors carry the document prefix. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/solr-mem-backfill/main.go | 2 +- cmd/solr-mem-server/embedding.go | 2 +- cmd/solr-mem-server/search_tool.go | 2 +- internal/embed/embed.go | 52 ++++++++++++++++++++++++------ internal/embed/embed_test.go | 36 +++++++++++++-------- 5 files changed, 68 insertions(+), 26 deletions(-) diff --git a/cmd/solr-mem-backfill/main.go b/cmd/solr-mem-backfill/main.go index b396b75..e376a2d 100644 --- a/cmd/solr-mem-backfill/main.go +++ b/cmd/solr-mem-backfill/main.go @@ -63,7 +63,7 @@ func main() { if id == "" || text == "" { continue } - vec, err := emb.Embed(ctx, text) + vec, err := emb.EmbedDocument(ctx, text) if err != nil || len(vec) == 0 { log.Printf("embed failed id=%s: %v", id, err) failed++ diff --git a/cmd/solr-mem-server/embedding.go b/cmd/solr-mem-server/embedding.go index 4e4f95d..43f6ed8 100644 --- a/cmd/solr-mem-server/embedding.go +++ b/cmd/solr-mem-server/embedding.go @@ -20,7 +20,7 @@ func embedMemoryText(ctx context.Context, title, content string) []float32 { if text == "" { return nil } - vec, err := embedder.Embed(ctx, text) + vec, err := embedder.EmbedDocument(ctx, text) if err != nil { log.Printf("embedding failed (proceeding without vector): %v", err) return nil diff --git a/cmd/solr-mem-server/search_tool.go b/cmd/solr-mem-server/search_tool.go index 7fa6cd3..31ef0b2 100644 --- a/cmd/solr-mem-server/search_tool.go +++ b/cmd/solr-mem-server/search_tool.go @@ -101,7 +101,7 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { // lexical-only on any embed/KNN error so search never hard-fails on the // optional path. if semantic { - if vec, eerr := embedder.Embed(ctx, query); eerr != nil { + if vec, eerr := embedder.EmbedQuery(ctx, query); eerr != nil { log.Printf("query embedding failed (lexical-only): %v", eerr) } else if len(vec) > 0 { knn, kerr := solrClient.KNNQuery(ctx, "embedding", vec, fusionK, params.FilterQueries, params.Fields) diff --git a/internal/embed/embed.go b/internal/embed/embed.go index f6abe93..eb17875 100644 --- a/internal/embed/embed.go +++ b/internal/embed/embed.go @@ -14,19 +14,28 @@ import ( "net/http" "os" "strconv" + "strings" "time" ) // Embedder turns text into a dense vector. Implementations must be safe for // concurrent use. +// +// Query and document embedding are distinct because some models (e.g. +// nomic-embed-text) are trained for asymmetric retrieval and expect different +// task prefixes on the stored text vs. the search query. Always embed stored +// memories with EmbedDocument and search queries with EmbedQuery so the two +// land in the same space. type Embedder interface { // Enabled reports whether embeddings are available. When false, callers // should fall back to lexical-only behavior. Enabled() bool // Dim is the vector dimension (must match the Solr DenseVectorField). Dim() int - // Embed returns the embedding for text. - Embed(ctx context.Context, text string) ([]float32, error) + // EmbedDocument embeds text to be stored/indexed. + EmbedDocument(ctx context.Context, text string) ([]float32, error) + // EmbedQuery embeds a search query. + EmbedQuery(ctx context.Context, text string) ([]float32, error) } // FromEnv builds an Embedder from EMBED_URL / EMBED_MODEL / EMBED_DIM. @@ -52,6 +61,18 @@ func FromEnv() Embedder { o.maxChars = n } } + // Task prefixes: default to nomic's for nomic models, off otherwise. + // EMBED_*_PREFIX env overrides either way (set to " " to force-clear). + if strings.Contains(strings.ToLower(model), "nomic") { + o.docPrefix = "search_document: " + o.queryPrefix = "search_query: " + } + if v, ok := os.LookupEnv("EMBED_DOC_PREFIX"); ok { + o.docPrefix = v + } + if v, ok := os.LookupEnv("EMBED_QUERY_PREFIX"); ok { + o.queryPrefix = v + } return o } @@ -60,17 +81,22 @@ type Disabled struct{} func (Disabled) Enabled() bool { return false } func (Disabled) Dim() int { return 0 } -func (Disabled) Embed(context.Context, string) ([]float32, error) { +func (Disabled) EmbedDocument(context.Context, string) ([]float32, error) { + return nil, nil +} +func (Disabled) EmbedQuery(context.Context, string) ([]float32, error) { return nil, nil } // Ollama embeds via an Ollama-compatible /api/embeddings endpoint. type Ollama struct { - baseURL string - model string - dim int - maxChars int // truncate input to this many runes (model context guard) - client *http.Client + baseURL string + model string + dim int + maxChars int // truncate input to this many runes (model context guard) + docPrefix string // task prefix for stored documents + queryPrefix string // task prefix for search queries + client *http.Client } // defaultMaxChars keeps embed input under typical small-model context windows @@ -92,7 +118,15 @@ func NewOllama(baseURL, model string, dim int) *Ollama { func (o *Ollama) Enabled() bool { return true } func (o *Ollama) Dim() int { return o.dim } -func (o *Ollama) Embed(ctx context.Context, text string) ([]float32, error) { +func (o *Ollama) EmbedDocument(ctx context.Context, text string) ([]float32, error) { + return o.embed(ctx, o.docPrefix+text) +} + +func (o *Ollama) EmbedQuery(ctx context.Context, text string) ([]float32, error) { + return o.embed(ctx, o.queryPrefix+text) +} + +func (o *Ollama) embed(ctx context.Context, text string) ([]float32, error) { text = truncateRunes(text, o.maxChars) body, err := json.Marshal(map[string]any{"model": o.model, "prompt": text}) if err != nil { diff --git a/internal/embed/embed_test.go b/internal/embed/embed_test.go index 38f8f89..679154d 100644 --- a/internal/embed/embed_test.go +++ b/internal/embed/embed_test.go @@ -15,13 +15,16 @@ func TestDisabled(t *testing.T) { if e.Enabled() { t.Fatal("Disabled must report not enabled") } - v, err := e.Embed(context.Background(), "x") - if err != nil || v != nil { - t.Fatalf("Disabled.Embed = %v, %v; want nil, nil", v, err) + if v, err := e.EmbedDocument(context.Background(), "x"); err != nil || v != nil { + t.Fatalf("Disabled.EmbedDocument = %v, %v; want nil, nil", v, err) + } + if v, err := e.EmbedQuery(context.Background(), "x"); err != nil || v != nil { + t.Fatalf("Disabled.EmbedQuery = %v, %v; want nil, nil", v, err) } } -func TestOllamaEmbed(t *testing.T) { +func TestOllamaEmbedAndPrefixes(t *testing.T) { + var gotPrompt string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/embeddings" { t.Errorf("path = %q", r.URL.Path) @@ -29,23 +32,28 @@ func TestOllamaEmbed(t *testing.T) { var req map[string]any b, _ := io.ReadAll(r.Body) json.Unmarshal(b, &req) - if req["model"] != "nomic-embed-text" || req["prompt"] != "hello" { - t.Errorf("unexpected request body: %v", req) - } + gotPrompt, _ = req["prompt"].(string) io.WriteString(w, `{"embedding":[0.1,0.2,0.3]}`) })) defer srv.Close() o := NewOllama(srv.URL, "nomic-embed-text", 3) - if !o.Enabled() || o.Dim() != 3 { - t.Fatalf("Enabled/Dim wrong: %v %d", o.Enabled(), o.Dim()) + o.docPrefix = "search_document: " + o.queryPrefix = "search_query: " + + v, err := o.EmbedDocument(context.Background(), "hello") + if err != nil || len(v) != 3 || v[0] != 0.1 { + t.Fatalf("EmbedDocument = %v, %v", v, err) } - v, err := o.Embed(context.Background(), "hello") - if err != nil { - t.Fatalf("Embed err: %v", err) + if gotPrompt != "search_document: hello" { + t.Errorf("doc prompt = %q, want prefixed", gotPrompt) + } + + if _, err := o.EmbedQuery(context.Background(), "hello"); err != nil { + t.Fatalf("EmbedQuery err: %v", err) } - if len(v) != 3 || v[0] != 0.1 { - t.Fatalf("vector = %v", v) + if gotPrompt != "search_query: hello" { + t.Errorf("query prompt = %q, want prefixed", gotPrompt) } } From e4c62ec7538cffa9320db5a3e1ef3804a22609ac Mon Sep 17 00:00:00 2001 From: arreyder Date: Wed, 10 Jun 2026 00:18:45 -0500 Subject: [PATCH 6/9] semantic search: upgrade to mxbai-embed-large (1024d) via new embedding1024 field Dimension change can't happen in place (Lucene forbids mixed vector dims in a field). Add embedding1024 (knn_vector_1024) and point store/update/backfill/KNN at it; the old 768 'embedding' field goes empty/vestigial (no delete-all, zero risk to existing data). Set EMBED_MODEL=mxbai-embed-large, EMBED_DIM=1024. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/solr-mem-backfill/main.go | 2 +- cmd/solr-mem-server/search_tool.go | 2 +- cmd/solr-mem-server/update_tool.go | 2 +- internal/solr/types.go | 2 +- solr/managed-schema.xml | 13 +++++++++---- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/solr-mem-backfill/main.go b/cmd/solr-mem-backfill/main.go index e376a2d..5a3fcae 100644 --- a/cmd/solr-mem-backfill/main.go +++ b/cmd/solr-mem-backfill/main.go @@ -71,7 +71,7 @@ func main() { } updates = append(updates, map[string]any{ "id": id, - "embedding": map[string]any{"set": vec}, + "embedding1024": map[string]any{"set": vec}, }) embedded++ } diff --git a/cmd/solr-mem-server/search_tool.go b/cmd/solr-mem-server/search_tool.go index 31ef0b2..75ab6d0 100644 --- a/cmd/solr-mem-server/search_tool.go +++ b/cmd/solr-mem-server/search_tool.go @@ -104,7 +104,7 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { if vec, eerr := embedder.EmbedQuery(ctx, query); eerr != nil { log.Printf("query embedding failed (lexical-only): %v", eerr) } else if len(vec) > 0 { - knn, kerr := solrClient.KNNQuery(ctx, "embedding", vec, fusionK, params.FilterQueries, params.Fields) + knn, kerr := solrClient.KNNQuery(ctx, "embedding1024", vec, fusionK, params.FilterQueries, params.Fields) if kerr != nil { log.Printf("knn search failed (lexical-only): %v", kerr) } else { diff --git a/cmd/solr-mem-server/update_tool.go b/cmd/solr-mem-server/update_tool.go index 929fc34..7a01ff2 100644 --- a/cmd/solr-mem-server/update_tool.go +++ b/cmd/solr-mem-server/update_tool.go @@ -91,7 +91,7 @@ func updateMemoryTool(ctx context.Context, args map[string]any) (any, error) { } } if vec := embedMemoryText(ctx, newTitle, newContent); vec != nil { - fields["embedding"] = vec + fields["embedding1024"] = vec } } diff --git a/internal/solr/types.go b/internal/solr/types.go index e6a003a..db87d04 100644 --- a/internal/solr/types.go +++ b/internal/solr/types.go @@ -22,7 +22,7 @@ type Document struct { Format string `json:"format,omitempty"` // Embedding is the dense semantic vector. Omitted when embeddings are // disabled so the field never reaches Solr in lexical-only mode. - Embedding []float32 `json:"embedding,omitempty"` + Embedding []float32 `json:"embedding1024,omitempty"` } // QueryParams holds parameters for a Solr search query. diff --git a/solr/managed-schema.xml b/solr/managed-schema.xml index 57cbd2e..4ee6107 100644 --- a/solr/managed-schema.xml +++ b/solr/managed-schema.xml @@ -9,9 +9,12 @@ - + + @@ -58,8 +61,10 @@ - + + From 5020757dd63c5add95b6b2cd3bc68e7ac00a681d Mon Sep 17 00:00:00 2001 From: arreyder Date: Wed, 10 Jun 2026 00:23:21 -0500 Subject: [PATCH 7/9] revert to nomic-embed-text: mxbai's 512-tok context truncated our corpus too hard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mxbai-embed-large (1024d) benchmarks higher but its 512-token window forced ~1200-char truncation (vs nomic's 6000) — embedding ~1/5 of each memory, with no conceptual-precision gain. nomic-embed-text (2048 tok) fits our content far better. Point store/update/backfill/KNN back at the 768 'embedding' field; embedding1024 stays defined-but-vestigial (kept so its data doesn't break reload). Env reverts to nomic/768/6000. Real conceptual-precision lever is reranking or chunked embeddings, not a bigger model. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/solr-mem-backfill/main.go | 2 +- cmd/solr-mem-server/search_tool.go | 2 +- cmd/solr-mem-server/update_tool.go | 2 +- internal/solr/types.go | 2 +- solr/managed-schema.xml | 6 ++++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/solr-mem-backfill/main.go b/cmd/solr-mem-backfill/main.go index 5a3fcae..e376a2d 100644 --- a/cmd/solr-mem-backfill/main.go +++ b/cmd/solr-mem-backfill/main.go @@ -71,7 +71,7 @@ func main() { } updates = append(updates, map[string]any{ "id": id, - "embedding1024": map[string]any{"set": vec}, + "embedding": map[string]any{"set": vec}, }) embedded++ } diff --git a/cmd/solr-mem-server/search_tool.go b/cmd/solr-mem-server/search_tool.go index 75ab6d0..31ef0b2 100644 --- a/cmd/solr-mem-server/search_tool.go +++ b/cmd/solr-mem-server/search_tool.go @@ -104,7 +104,7 @@ func searchMemoriesTool(ctx context.Context, args map[string]any) (any, error) { if vec, eerr := embedder.EmbedQuery(ctx, query); eerr != nil { log.Printf("query embedding failed (lexical-only): %v", eerr) } else if len(vec) > 0 { - knn, kerr := solrClient.KNNQuery(ctx, "embedding1024", vec, fusionK, params.FilterQueries, params.Fields) + knn, kerr := solrClient.KNNQuery(ctx, "embedding", vec, fusionK, params.FilterQueries, params.Fields) if kerr != nil { log.Printf("knn search failed (lexical-only): %v", kerr) } else { diff --git a/cmd/solr-mem-server/update_tool.go b/cmd/solr-mem-server/update_tool.go index 7a01ff2..929fc34 100644 --- a/cmd/solr-mem-server/update_tool.go +++ b/cmd/solr-mem-server/update_tool.go @@ -91,7 +91,7 @@ func updateMemoryTool(ctx context.Context, args map[string]any) (any, error) { } } if vec := embedMemoryText(ctx, newTitle, newContent); vec != nil { - fields["embedding1024"] = vec + fields["embedding"] = vec } } diff --git a/internal/solr/types.go b/internal/solr/types.go index db87d04..e6a003a 100644 --- a/internal/solr/types.go +++ b/internal/solr/types.go @@ -22,7 +22,7 @@ type Document struct { Format string `json:"format,omitempty"` // Embedding is the dense semantic vector. Omitted when embeddings are // disabled so the field never reaches Solr in lexical-only mode. - Embedding []float32 `json:"embedding1024,omitempty"` + Embedding []float32 `json:"embedding,omitempty"` } // QueryParams holds parameters for a Solr search query. diff --git a/solr/managed-schema.xml b/solr/managed-schema.xml index 4ee6107..272c1d9 100644 --- a/solr/managed-schema.xml +++ b/solr/managed-schema.xml @@ -62,8 +62,10 @@ + responses; indexed=true powers KNN. embedding=768/nomic-embed-text + (current, used by search — its 2048-tok context fits our memories). + embedding1024=mxbai-embed-large (vestigial: 512-tok context truncated too + hard on this corpus; kept defined so existing data doesn't break reload). --> From fa38b5a33f1ff11e0741bff2b5fd4cc3a62e33dd Mon Sep 17 00:00:00 2001 From: arreyder Date: Thu, 13 Aug 2026 10:52:33 -0500 Subject: [PATCH 8/9] fix: stop EnsureCollection from destroying a corrupt core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code core went down on Aug 9 when an unclean shutdown truncated two segment files mid-flush (46k docs across 25 segments; the 1.3GB main segment was fine). That alone was recoverable — the index was intact on disk and CheckIndex -exorcise brings it back. What made it unrecoverable was the retry path. A core that fails to load answers every request with a 500, so Ping fails, so EnsureCollection concluded the core was missing and fired CoreAdmin CREATE. Solr deletes the core.properties of a failed CREATE, which unregistered the core and orphaned the index. The indexer then crash-looped under launchd, firing that same destructive CREATE every 10s for three days. EnsureCollection now checks CoreAdmin STATUS before creating anything and only CREATEs a core Solr has genuinely never heard of. A core listed under initFailures returns a descriptive error pointing at the repair procedure. Also fixed along the way: - code-solrconfig.xml used solr.TieredMergePolicyFactory, which does not resolve — it needs the fully qualified name. This never surfaced because a core reads config from its own instanceDir/conf, so the May 30 tuning had never actually been loaded by the running core. - Mount the configsets at $SOLR_HOME/configsets, where Solr resolves configSet= names. The existing /opt/solr mounts only feed precreate-core, so any CREATE naming a configSet failed with "Could not load configuration from directory /var/solr/data/configsets/code". - Document the repair procedure and the instanceDir config-drift trap. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 49 ++++++++++++++++ docker-compose.yml | 12 ++++ internal/solr/client.go | 75 +++++++++++++++++++++++++ internal/solr/client_test.go | 106 +++++++++++++++++++++++++++++++++++ solr/code-solrconfig.xml | 4 +- 5 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 internal/solr/client_test.go diff --git a/README.md b/README.md index 15cc694..efa84c5 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,55 @@ The indexer manages its own clones in `~/solr-mem-repos/` and polls for new comm Logs: `/tmp/solr-mem-server.log` and `/tmp/solr-mem-indexer.log` +## Recovering a corrupt core + +An unclean shutdown (host sleep, `colima stop`, OOM kill) can truncate segment +files mid-flush and leave a core that will not load. Solr reports it as +`SolrCore 'code' is not available due to init failure: Error opening new +searcher`, and `admin/cores?action=STATUS` lists the core under `initFailures`. + +**Do not run CoreAdmin CREATE against a core in this state.** Solr deletes the +`core.properties` of a failed CREATE, which unregisters the core and orphans a +perfectly recoverable index on disk. `EnsureCollection` checks STATUS and +refuses for this reason; a hand-run `curl` has no such guard. + +Diagnose which segments are broken (read-only, safe to run anytime): + +```bash +docker exec solr-mem bash -lc 'cd /opt/solr && java \ + -cp "server/solr-webapp/webapp/WEB-INF/lib/*:server/lib/ext/*" \ + org.apache.lucene.index.CheckIndex /var/solr/data/code/data/index -fast' +``` + +If it reports broken segments, drop them. This loses only the documents in +those segments — the indexer re-adds them on its next pass: + +```bash +# Stop writers first so nothing holds the index lock. +launchctl bootout gui/$UID/com.solr-mem.indexer + +docker exec solr-mem bash -lc 'cd /opt/solr && java \ + -cp "server/solr-webapp/webapp/WEB-INF/lib/*:server/lib/ext/*" \ + org.apache.lucene.index.CheckIndex /var/solr/data/code/data/index -exorcise' + +# Re-register the core. instanceDir has its own conf/, so pass no configSet. +curl "http://localhost:8983/solr/admin/cores?action=CREATE&name=code&instanceDir=code" + +launchctl bootstrap gui/$UID ~/Library/LaunchAgents/com.solr-mem.indexer.plist +``` + +Back up the memories core before any repair work — it is the only collection +that cannot be rebuilt from source: + +```bash +curl "http://localhost:8983/solr/memories/replication?command=backup&location=/var/solr/data&name=safety" +``` + +Note that a core's config lives in its own `instanceDir/conf`, copied there +once at creation. Editing `solr/*.xml` in this repo does **not** reach an +existing core — copy the files in and reload the core, or the core keeps +running the config it was born with. + ## Architecture ``` diff --git a/docker-compose.yml b/docker-compose.yml index 6b7cd1e..4dbbf33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,18 @@ services: - ./solr/stopwords.txt:/opt/solr/server/solr/configsets/code/conf/stopwords.txt - ./solr/synonyms.txt:/opt/solr/server/solr/configsets/code/conf/synonyms.txt - solr_data:/var/solr + # Solr resolves `configSet=` against $SOLR_HOME/configsets, i.e. + # /var/solr/data/configsets — NOT the /opt/solr configsets above, which + # only feed precreate-core. Without these, any CoreAdmin CREATE that names + # a configSet fails with "Could not load configuration from directory". + - ./solr/managed-schema.xml:/var/solr/data/configsets/memories/conf/managed-schema.xml + - ./solr/solrconfig.xml:/var/solr/data/configsets/memories/conf/solrconfig.xml + - ./solr/stopwords.txt:/var/solr/data/configsets/memories/conf/stopwords.txt + - ./solr/synonyms.txt:/var/solr/data/configsets/memories/conf/synonyms.txt + - ./solr/code-managed-schema.xml:/var/solr/data/configsets/code/conf/managed-schema.xml + - ./solr/code-solrconfig.xml:/var/solr/data/configsets/code/conf/solrconfig.xml + - ./solr/stopwords.txt:/var/solr/data/configsets/code/conf/stopwords.txt + - ./solr/synonyms.txt:/var/solr/data/configsets/code/conf/synonyms.txt environment: SOLR_JAVA_MEM: "-Xms1g -Xmx4g" # Cap a little above the 4g JVM heap so Docker enforces a bound that leaves diff --git a/internal/solr/client.go b/internal/solr/client.go index a98aab5..5faf829 100644 --- a/internal/solr/client.go +++ b/internal/solr/client.go @@ -394,9 +394,69 @@ func (c *Client) DeleteByQuery(ctx context.Context, query string) error { return c.checkResponse(resp) } +// coreState describes what Solr knows about a core: whether it is registered, +// and whether it is registered but broken. +type coreState struct { + exists bool + initFailed bool + initError string +} + +// coreStatus asks the CoreAdmin API about a single core. A core that Solr has +// never heard of comes back as an empty status entry; a core whose index or +// config is broken comes back under initFailures. The two cases need very +// different handling, so they are reported separately. +func (c *Client) coreStatus(ctx context.Context, solrBase, core string) (coreState, error) { + statusURL := fmt.Sprintf("%s/solr/admin/cores?action=STATUS&core=%s&wt=json", + solrBase, url.QueryEscape(core)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil) + if err != nil { + return coreState{}, fmt.Errorf("core status request: %w", err) + } + resp, err := c.httpClient.Do(req) + if err != nil { + return coreState{}, fmt.Errorf("core status: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return coreState{}, fmt.Errorf("core status returned %d: %s", resp.StatusCode, body) + } + + var out struct { + InitFailures map[string]json.RawMessage `json:"initFailures"` + Status map[string]struct { + Name string `json:"name"` + InstanceDir string `json:"instanceDir"` + } `json:"status"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return coreState{}, fmt.Errorf("decode core status: %w", err) + } + + state := coreState{} + if raw, ok := out.InitFailures[core]; ok { + state.initFailed = true + state.initError = strings.Trim(string(raw), `"`) + } + if st, ok := out.Status[core]; ok && st.Name != "" { + state.exists = true + } + return state, nil +} + // EnsureCollection creates the Solr collection if it doesn't already exist. // It uses the configDir to upload the schema/config files. // baseURL should be like "http://host:8983/solr/code" — the collection name is extracted from the path. +// +// A failing ping is not on its own proof that the core is missing: a core whose +// index is corrupt answers every request with a 500 while its data sits intact +// on disk. Issuing CREATE against that core makes things strictly worse — +// Solr deletes the core.properties of a failed CREATE, which unregisters the +// core entirely and leaves the index orphaned. So we check CoreAdmin STATUS +// and only CREATE a core Solr has genuinely never heard of. func (c *Client) EnsureCollection(ctx context.Context, configDir string) error { // Check if collection already exists via ping if err := c.Ping(ctx); err == nil { @@ -411,6 +471,21 @@ func (c *Client) EnsureCollection(ctx context.Context, configDir string) error { collection := path.Base(u.Path) solrBase := strings.TrimSuffix(c.baseURL, "/solr/"+collection) + state, err := c.coreStatus(ctx, solrBase, collection) + if err != nil { + return fmt.Errorf("determine state of core %s: %w", collection, err) + } + switch { + case state.initFailed: + return fmt.Errorf("core %s exists but failed to initialize: %s\n"+ + "refusing to CREATE over it — that would delete its core.properties and orphan the index. "+ + "Repair the core (see README: recovering a corrupt core) and reload it", + collection, state.initError) + case state.exists: + // Registered, but not answering ping yet — still loading or warming. + return nil + } + // Create collection using the Solr ConfigSet API + Collections API // First, try creating via the core admin API (standalone mode, not SolrCloud) createURL := fmt.Sprintf("%s/solr/admin/cores?action=CREATE&name=%s&instanceDir=%s&configSet=%s", diff --git a/internal/solr/client_test.go b/internal/solr/client_test.go new file mode 100644 index 0000000..e84127f --- /dev/null +++ b/internal/solr/client_test.go @@ -0,0 +1,106 @@ +package solr + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// fakeSolr serves /admin/ping and /admin/cores?action=STATUS for the core +// named "code", and records whether a CREATE was ever attempted. +type fakeSolr struct { + pingStatus int // status code returned by /solr/code/admin/ping + statusJSON string // body returned by /solr/admin/cores?action=STATUS + createCalls int +} + +func (f *fakeSolr) start(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/solr/code/admin/ping", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(f.pingStatus) + }) + mux.HandleFunc("/solr/admin/cores", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("action") == "CREATE" { + f.createCalls++ + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(f.statusJSON)) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +// A core that is registered but failed to load answers every request with a +// 500. CREATEing over it deletes its core.properties and orphans the index on +// disk, so EnsureCollection must refuse. +func TestEnsureCollection_RefusesCreateOnInitFailure(t *testing.T) { + f := &fakeSolr{ + pingStatus: http.StatusInternalServerError, + statusJSON: `{"initFailures":{"code":"Error opening new searcher"},"status":{"code":{}}}`, + } + srv := f.start(t) + + err := NewClient(srv.URL + "/solr/code").EnsureCollection(context.Background(), "") + if err == nil { + t.Fatal("expected an error for a core that failed to initialize") + } + if !strings.Contains(err.Error(), "Error opening new searcher") { + t.Errorf("error should surface Solr's init failure, got: %v", err) + } + if f.createCalls != 0 { + t.Errorf("must not CREATE over a broken core, got %d CREATE calls", f.createCalls) + } +} + +// A core Solr has never heard of comes back as an empty status entry; that is +// the one case where CREATE is the right move. +func TestEnsureCollection_CreatesMissingCore(t *testing.T) { + f := &fakeSolr{ + pingStatus: http.StatusInternalServerError, + statusJSON: `{"initFailures":{},"status":{"code":{}}}`, + } + srv := f.start(t) + + if err := NewClient(srv.URL + "/solr/code").EnsureCollection(context.Background(), ""); err != nil { + t.Fatalf("EnsureCollection: %v", err) + } + if f.createCalls != 1 { + t.Errorf("expected exactly 1 CREATE for a missing core, got %d", f.createCalls) + } +} + +// Registered but not yet answering ping (loading/warming) is not a reason to +// create anything. +func TestEnsureCollection_LeavesLoadingCoreAlone(t *testing.T) { + f := &fakeSolr{ + pingStatus: http.StatusServiceUnavailable, + statusJSON: `{"initFailures":{},"status":{"code":{"name":"code","instanceDir":"/var/solr/data/code"}}}`, + } + srv := f.start(t) + + if err := NewClient(srv.URL + "/solr/code").EnsureCollection(context.Background(), ""); err != nil { + t.Fatalf("EnsureCollection: %v", err) + } + if f.createCalls != 0 { + t.Errorf("must not CREATE over a registered core, got %d CREATE calls", f.createCalls) + } +} + +// A healthy core short-circuits on ping without touching the admin API. +func TestEnsureCollection_HealthyCoreIsNoop(t *testing.T) { + f := &fakeSolr{pingStatus: http.StatusOK} + srv := f.start(t) + + if err := NewClient(srv.URL + "/solr/code").EnsureCollection(context.Background(), ""); err != nil { + t.Fatalf("EnsureCollection: %v", err) + } + if f.createCalls != 0 { + t.Errorf("healthy core must not trigger CREATE, got %d", f.createCalls) + } +} diff --git a/solr/code-solrconfig.xml b/solr/code-solrconfig.xml index f7d8dac..d0e5dfe 100644 --- a/solr/code-solrconfig.xml +++ b/solr/code-solrconfig.xml @@ -13,7 +13,9 @@ ${solr.lock.type:native} 256 - + + 6 6 5.0 From bb782b2b48497279c2982f8605a1fbb6bbb9ac0a Mon Sep 17 00:00:00 2001 From: arreyder Date: Mon, 17 Aug 2026 08:07:08 -0500 Subject: [PATCH 9/9] fix(parser): resolve physical line numbers, not //line-adjusted ones The indexer was crash-looping on every pass: panic: runtime error: slice bounds out of range [1000005:21] parser.extractLines internal/parser/go_parser.go:580 parser.(*GoParser).extractFunc go_parser.go:72 go/token's Position() honors //line directives, which rewrite the line numbers it reports. The vendored Go SDK ships cmd/objdump/testdata/fmthello.go containing "//line fmthello.go:999999" on line 8, so `func Println` on physical line 16 was reported as line 1000006. We slice the physical file content, so extractLines got a start of 1000006 against 21 lines and panicked. extractLines clamped end down to len(lines) and start up to 1, but never start down, so start-1 > end produced an invalid slice range. Fixed both layers: - Resolve positions with PositionFor(pos, false) to get unadjusted, physical lines. This is a correctness fix as much as a crash fix: any file with a //line directive previously got wrong LineStart/LineEnd and a misaligned body. - Clamp extractLines on both ends and return "" for empty/inverted ranges. This indexes whatever happens to be vendored into a repo, so one pathological file must not kill a run over a million others. Test reproduces the panic exactly (verified failing without the fix) and covers the bounds cases. Co-Authored-By: Claude Opus 5 (1M context) --- internal/parser/go_parser.go | 31 +++++++-- internal/parser/go_parser_test.go | 100 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 internal/parser/go_parser_test.go diff --git a/internal/parser/go_parser.go b/internal/parser/go_parser.go index d610d13..bc674fa 100644 --- a/internal/parser/go_parser.go +++ b/internal/parser/go_parser.go @@ -62,8 +62,8 @@ func (p *GoParser) Parse(filePath string, content []byte) (*FileInfo, error) { } func (p *GoParser) extractFunc(fset *token.FileSet, fn *ast.FuncDecl, lines []string) Symbol { - startLine := fset.Position(fn.Pos()).Line - endLine := fset.Position(fn.End()).Line + startLine := physicalLine(fset, fn.Pos()) + endLine := physicalLine(fset, fn.End()) sym := Symbol{ Name: fn.Name.Name, @@ -106,8 +106,8 @@ func (p *GoParser) extractGenDecl(fset *token.FileSet, gd *ast.GenDecl, lines [] for _, spec := range gd.Specs { switch s := spec.(type) { case *ast.TypeSpec: - startLine := fset.Position(gd.Pos()).Line - endLine := fset.Position(gd.End()).Line + startLine := physicalLine(fset, gd.Pos()) + endLine := physicalLine(fset, gd.End()) sym := Symbol{ Name: s.Name.Name, @@ -137,8 +137,8 @@ func (p *GoParser) extractGenDecl(fset *token.FileSet, gd *ast.GenDecl, lines [] syms = append(syms, sym) case *ast.ValueSpec: - startLine := fset.Position(s.Pos()).Line - endLine := fset.Position(s.End()).Line + startLine := physicalLine(fset, s.Pos()) + endLine := physicalLine(fset, s.End()) for _, name := range s.Names { sym := Symbol{ @@ -570,12 +570,31 @@ func exprString(expr ast.Expr) string { } } +// physicalLine reports the line number a position occupies in the file as it +// exists on disk, ignoring any //line directives. go/token's Position() honors +// those directives, which can report a line far outside the file — vendored Go +// SDK testdata contains "//line fmthello.go:999999" — and we index the physical +// bytes, so an adjusted line number would not address the right text. +func physicalLine(fset *token.FileSet, pos token.Pos) int { + return fset.PositionFor(pos, false).Line +} + +// extractLines returns lines[start:end] as text, 1-indexed and inclusive of +// end. Bounds are clamped rather than trusted: this runs over whatever source +// happens to be vendored into a repo, and one pathological file must not take +// down a batch indexing a million others. func extractLines(lines []string, start, end int) string { if start < 1 { start = 1 } + if start > len(lines) { + return "" + } if end > len(lines) { end = len(lines) } + if end < start { + return "" + } return strings.Join(lines[start-1:end], "\n") } diff --git a/internal/parser/go_parser_test.go b/internal/parser/go_parser_test.go new file mode 100644 index 0000000..814eaad --- /dev/null +++ b/internal/parser/go_parser_test.go @@ -0,0 +1,100 @@ +package parser + +import ( + "strings" + "testing" +) + +// Shape taken from the Go SDK's cmd/objdump/testdata/fmthello.go, which is +// vendored into real repos. The //line directive remaps every position after +// it to ~1e6, far past the end of a 20-line file. +const lineDirectiveSrc = `package main + +import "fmt" + +func main() { + Println("hello, world") + if flag { +//line fmthello.go:999999 + Println("bad line") + for { + } + } +} + +//go:noinline +func Println(s string) { + fmt.Println(s) +} + +var flag bool +` + +// Before the fix this panicked with "slice bounds out of range [1000005:21]" +// and killed the whole indexing run. +func TestParse_LineDirectiveBeyondEOF(t *testing.T) { + p := &GoParser{} + info, err := p.Parse("fmthello.go", []byte(lineDirectiveSrc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + total := len(strings.Split(lineDirectiveSrc, "\n")) + byName := map[string]Symbol{} + for _, s := range info.Symbols { + byName[s.Name] = s + } + + // Every symbol must land inside the physical file. + for _, s := range info.Symbols { + if s.LineStart < 1 || s.LineStart > total { + t.Errorf("%s: LineStart %d outside file of %d lines", s.Name, s.LineStart, total) + } + if s.LineEnd < s.LineStart || s.LineEnd > total { + t.Errorf("%s: LineEnd %d invalid (start %d, %d lines)", s.Name, s.LineEnd, s.LineStart, total) + } + } + + // Println is declared after the directive; its physical line is 16, and the + // body we slice out must be the real source text, not empty or misaligned. + println, ok := byName["Println"] + if !ok { + t.Fatalf("Println not extracted; got %d symbols", len(info.Symbols)) + } + if println.LineStart != 16 { + t.Errorf("Println LineStart = %d, want 16 (physical line)", println.LineStart) + } + if !strings.Contains(println.Body, "func Println(s string)") { + t.Errorf("Println body does not contain its own declaration: %q", println.Body) + } +} + +func TestExtractLines_Bounds(t *testing.T) { + lines := []string{"a", "b", "c"} + + tests := []struct { + name string + start, end int + want string + }{ + {"normal range", 1, 2, "a\nb"}, + {"single line", 2, 2, "b"}, + {"end past EOF clamps", 2, 99, "b\nc"}, + {"start below 1 clamps", -5, 1, "a"}, + {"start past EOF is empty", 99, 100, ""}, + {"inverted range is empty", 3, 1, ""}, + {"empty input", 1, 1, ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := lines + if tc.name == "empty input" { + in = nil + } + if got := extractLines(in, tc.start, tc.end); got != tc.want { + t.Errorf("extractLines(%d, %d) = %q, want %q", tc.start, tc.end, got, tc.want) + } + }) + } +}