fix(index): refresh stale auto-discovered caches

CHANGELOG.md

@@ -12,6 +12,8 @@ All notable changes to this project are documented in this file.
   It reports the index scope in text and JSON output.
 
 ### Fixed
+- CLI and MCP searches now refresh stale auto-discovered indexes after source edits, deletions, or additions.
+- `index validate` now detects added source files and returns status 2 for stale JSON reports.
 - Large full-index runs no longer rely on host CPU count and optional GC tuning for memory containment.
 - Minified JavaScript bundles are now classified before structural parsing.
 - Complexity analysis now reuses source spans and processes files in parallel.

README.md

@@ -41,6 +41,8 @@ Current main uses gotreesitter `v0.45.0` and keeps large-repository indexing bou
 
 Diff-aware checks and reviews use a valid repository index when one exists.
 Without one, they build and report a changed-only snapshot.
+Auto-discovered indexes refresh incrementally when source metadata changes.
+An explicit `--cache` path remains a fixed snapshot.
 
 The defaults remain tunable for unusual workloads:
 

cmd/canopy/helpers.go

@@ -93,25 +93,29 @@ func loadOrBuildWithScope(cmd *cobra.Command, cachePath string, target string, n
 	if !noCache {
 		autoPath := filepath.Join(target, ".canopy", "index.json")
 		if fi, err := os.Stat(autoPath); err == nil {
-			// Use lenient load: accept older schema versions with a warning
-			// rather than rebuilding from scratch (which loads all grammars
-			// and can OOM on large repos).
 			if idx, loadErr := index.LoadLenient(autoPath); loadErr == nil {
-				age := time.Since(fi.ModTime()).Truncate(time.Second)
-				if idx.ConfigHashes == nil {
-					fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, rebuild with 'gts index build' for config tracking)\n", autoPath, age)
-					return idx.ExcludePaths(excludes), false, nil
+				builder, buildErr := index.NewBuilderWithWorkspaceIgnores(target)
+				if buildErr != nil {
+					return nil, false, buildErr
+				}
+				idx, status, refreshErr := builder.EnsureFreshCache(cmd.Context(), target, autoPath, idx)
+				if refreshErr != nil {
+					return nil, false, refreshErr
 				}
-				current, hashErr := index.ComputeConfigHashes(target)
-				if hashErr == nil && configHashesMatch(idx.ConfigHashes, current) {
+				age := time.Since(fi.ModTime()).Truncate(time.Second)
+				switch status {
+				case index.CacheIncrementallyRefreshed:
+					fmt.Fprintf(os.Stderr, "index: refreshed stale cache %s\n", autoPath)
+				case index.CacheFullyRebuilt:
+					fmt.Fprintf(os.Stderr, "index: rebuilt cache %s after configuration or root change\n", autoPath)
+				default:
 					if len(excludes) > 0 {
 						fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, applying %d exclusion patterns post-load)\n", autoPath, age, len(excludes))
 					} else {
-						fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, pass --no-cache for fresh)\n", autoPath, age)
+						fmt.Fprintf(os.Stderr, "index: using fresh cache %s (age %s)\n", autoPath, age)
 					}
-					return idx.ExcludePaths(excludes), false, nil
 				}
-				fmt.Fprintf(os.Stderr, "index: config changed since last build, rebuilding...\n")
+				return idx.ExcludePaths(excludes), false, nil
 			}
 		}
 	}
@@ -129,18 +133,6 @@ func loadOrBuildWithScope(cmd *cobra.Command, cachePath string, target string, n
 	return idx, false, err
 }
 
-func configHashesMatch(cached, current map[string]string) bool {
-	if len(cached) != len(current) {
-		return false
-	}
-	for k, v := range cached {
-		if current[k] != v {
-			return false
-		}
-	}
-	return true
-}
-
 func emitJSON(value any) error {
 	encoder := json.NewEncoder(os.Stdout)
 	encoder.SetIndent("", "  ")

cmd/canopy/helpers_test.go

@@ -7,6 +7,9 @@ import (
 	"testing"
 
 	"github.com/spf13/cobra"
+
+	"m31labs.dev/canopy/pkg/index"
+	"m31labs.dev/canopy/pkg/model"
 )
 
 func TestLoadOrBuildChangedUsesChangedOnlyFallback(t *testing.T) {
@@ -75,3 +78,51 @@ func TestLoadOrBuildCheckDerivesChangedPathsFromBase(t *testing.T) {
 		t.Fatalf("check path = %q, want %q", got, want)
 	}
 }
+
+func TestLoadOrBuildRefreshesAutoCacheWithoutConfig(t *testing.T) {
+	root := t.TempDir()
+	sourcePath := filepath.Join(root, "main.go")
+	if err := os.WriteFile(sourcePath, []byte("package sample\n\nfunc Cached() {}\n"), 0o644); err != nil {
+		t.Fatalf("WriteFile(main.go) failed: %v", err)
+	}
+	builder, err := index.NewBuilderWithWorkspaceIgnores(root)
+	if err != nil {
+		t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err)
+	}
+	cached, err := builder.BuildPath(root)
+	if err != nil {
+		t.Fatalf("BuildPath returned error: %v", err)
+	}
+	cachePath := filepath.Join(root, ".canopy", "index.json")
+	if err := index.Save(cachePath, cached); err != nil {
+		t.Fatalf("Save returned error: %v", err)
+	}
+	if cached.ConfigHashes != nil {
+		t.Fatalf("ConfigHashes = %#v, want nil for this regression", cached.ConfigHashes)
+	}
+
+	if err := os.WriteFile(sourcePath, []byte("package sample\n\nfunc RefreshedWithLongerName() {}\n"), 0o644); err != nil {
+		t.Fatalf("WriteFile edited main.go failed: %v", err)
+	}
+	loaded, err := loadOrBuild(&cobra.Command{}, "", root, false)
+	if err != nil {
+		t.Fatalf("loadOrBuild returned error: %v", err)
+	}
+	if !helperIndexHasSymbol(loaded, "RefreshedWithLongerName") {
+		t.Fatal("auto-discovered cache did not refresh the edited source")
+	}
+	if helperIndexHasSymbol(loaded, "Cached") {
+		t.Fatal("auto-discovered cache retained the stale symbol")
+	}
+}
+
+func helperIndexHasSymbol(idx *model.Index, name string) bool {
+	for _, file := range idx.Files {
+		for _, symbol := range file.Symbols {
+			if symbol.Name == name {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/canopy/index_validate.go

@@ -1,23 +1,27 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
-	"os"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 
 	"m31labs.dev/canopy/pkg/index"
+	"m31labs.dev/canopy/pkg/model"
 )
 
 type validateReport struct {
-	Total       int      `json:"total"`
-	OK          int      `json:"ok"`
-	Stale       int      `json:"stale"`
-	Missing     int      `json:"missing"`
-	ParseErrors int      `json:"parse_errors"`
-	StaleFiles  []string `json:"stale_files,omitempty"`
+	Total        int      `json:"total"`
+	OK           int      `json:"ok"`
+	Stale        int      `json:"stale"`
+	Missing      int      `json:"missing"`
+	New          int      `json:"new"`
+	ParseErrors  int      `json:"parse_errors"`
+	RootMismatch bool     `json:"root_mismatch,omitempty"`
+	StaleFiles   []string `json:"stale_files,omitempty"`
 	MissingFiles []string `json:"missing_files,omitempty"`
+	NewFiles     []string `json:"new_files,omitempty"`
 }
 
 func newValidateCmd() *cobra.Command {
@@ -26,7 +30,7 @@ func newValidateCmd() *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "validate [path]",
-		Short: "Check index integrity and detect stale or missing files",
+		Short: "Check index integrity and detect stale, missing, or new files",
 		Args:  cobra.MaximumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if cachePath == "" {
@@ -43,55 +47,28 @@ func newValidateCmd() *cobra.Command {
 				root = args[0]
 			}
 
-			report := validateReport{
-				Total:       len(idx.Files),
-				ParseErrors: len(idx.Errors),
-			}
-
-			for _, f := range idx.Files {
-				absPath := f.Path
-				if !filepath.IsAbs(absPath) {
-					absPath = filepath.Join(root, absPath)
-				}
-
-				info, err := os.Stat(absPath)
-				if err != nil {
-					report.Missing++
-					report.MissingFiles = append(report.MissingFiles, f.Path)
-					continue
-				}
-
-				if info.ModTime().After(idx.GeneratedAt) {
-					report.Stale++
-					report.StaleFiles = append(report.StaleFiles, f.Path)
-					continue
-				}
-
-				report.OK++
+			report, err := buildValidateReport(cmd.Context(), root, idx)
+			if err != nil {
+				return err
 			}
 
 			if jsonOutput {
-				return emitJSON(report)
-			}
-
-			fmt.Printf("validate: total=%d ok=%d stale=%d missing=%d parse_errors=%d\n",
-				report.Total, report.OK, report.Stale, report.Missing, report.ParseErrors)
-
-			if len(report.MissingFiles) > 0 {
-				fmt.Println("missing:")
-				for _, p := range report.MissingFiles {
-					fmt.Printf("  %s\n", p)
-				}
-			}
-			if len(report.StaleFiles) > 0 {
-				fmt.Println("stale:")
-				for _, p := range report.StaleFiles {
-					fmt.Printf("  %s\n", p)
+				if err := emitJSON(report); err != nil {
+					return err
 				}
+			} else {
+				fmt.Printf("validate: total=%d ok=%d stale=%d missing=%d new=%d parse_errors=%d root_mismatch=%t\n",
+					report.Total, report.OK, report.Stale, report.Missing, report.New, report.ParseErrors, report.RootMismatch)
+				printValidateFiles("missing", report.MissingFiles)
+				printValidateFiles("stale", report.StaleFiles)
+				printValidateFiles("new", report.NewFiles)
 			}
 
-			if report.Stale > 0 || report.Missing > 0 {
-				os.Exit(2)
+			if report.RootMismatch || report.Stale > 0 || report.Missing > 0 || report.New > 0 {
+				return exitCodeError{
+					code: 2,
+					err:  errors.New("index source set is stale"),
+				}
 			}
 			return nil
 		},
@@ -101,3 +78,51 @@ func newValidateCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&jsonOutput, "json", false, "emit JSON output")
 	return cmd
 }
+
+func buildValidateReport(ctx context.Context, root string, idx *model.Index) (validateReport, error) {
+	builder, err := index.NewBuilderWithWorkspaceIgnores(root)
+	if err != nil {
+		return validateReport{}, err
+	}
+	freshness, err := builder.CheckFreshness(ctx, root, idx)
+	if err != nil {
+		return validateReport{}, err
+	}
+
+	report := validateReport{
+		Total:        len(idx.Files),
+		OK:           len(idx.Files),
+		Stale:        len(freshness.StaleFiles),
+		Missing:      len(freshness.MissingFiles),
+		New:          len(freshness.NewFiles),
+		ParseErrors:  len(idx.Errors),
+		RootMismatch: freshness.RootMismatch,
+		StaleFiles:   freshness.StaleFiles,
+		MissingFiles: freshness.MissingFiles,
+		NewFiles:     freshness.NewFiles,
+	}
+	if freshness.RootMismatch {
+		report.OK = 0
+		return report, nil
+	}
+	indexedPaths := make(map[string]struct{}, len(idx.Files))
+	for _, file := range idx.Files {
+		indexedPaths[file.Path] = struct{}{}
+	}
+	for _, path := range append(freshness.StaleFiles, freshness.MissingFiles...) {
+		if _, ok := indexedPaths[path]; ok {
+			report.OK--
+		}
+	}
+	return report, nil
+}
+
+func printValidateFiles(label string, paths []string) {
+	if len(paths) == 0 {
+		return
+	}
+	fmt.Printf("%s:\n", label)
+	for _, path := range paths {
+		fmt.Printf("  %s\n", path)
+	}
+}