diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c69397..13f7958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes to this project are documented in this file. It reports the index scope in text and JSON output. ### Fixed +- CLI and MCP searches now refresh stale auto-discovered indexes after source edits, deletions, or additions. +- `index validate` now detects added source files and returns status 2 for stale JSON reports. - Large full-index runs no longer rely on host CPU count and optional GC tuning for memory containment. - Minified JavaScript bundles are now classified before structural parsing. - Complexity analysis now reuses source spans and processes files in parallel. diff --git a/README.md b/README.md index 8e4fd12..d87255f 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ Current main uses gotreesitter `v0.45.0` and keeps large-repository indexing bou Diff-aware checks and reviews use a valid repository index when one exists. Without one, they build and report a changed-only snapshot. +Auto-discovered indexes refresh incrementally when source metadata changes. +An explicit `--cache` path remains a fixed snapshot. The defaults remain tunable for unusual workloads: diff --git a/cmd/canopy/helpers.go b/cmd/canopy/helpers.go index e5efcc7..56ad0d3 100644 --- a/cmd/canopy/helpers.go +++ b/cmd/canopy/helpers.go @@ -93,25 +93,29 @@ func loadOrBuildWithScope(cmd *cobra.Command, cachePath string, target string, n if !noCache { autoPath := filepath.Join(target, ".canopy", "index.json") if fi, err := os.Stat(autoPath); err == nil { - // Use lenient load: accept older schema versions with a warning - // rather than rebuilding from scratch (which loads all grammars - // and can OOM on large repos). if idx, loadErr := index.LoadLenient(autoPath); loadErr == nil { - age := time.Since(fi.ModTime()).Truncate(time.Second) - if idx.ConfigHashes == nil { - fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, rebuild with 'gts index build' for config tracking)\n", autoPath, age) - return idx.ExcludePaths(excludes), false, nil + builder, buildErr := index.NewBuilderWithWorkspaceIgnores(target) + if buildErr != nil { + return nil, false, buildErr + } + idx, status, refreshErr := builder.EnsureFreshCache(cmd.Context(), target, autoPath, idx) + if refreshErr != nil { + return nil, false, refreshErr } - current, hashErr := index.ComputeConfigHashes(target) - if hashErr == nil && configHashesMatch(idx.ConfigHashes, current) { + age := time.Since(fi.ModTime()).Truncate(time.Second) + switch status { + case index.CacheIncrementallyRefreshed: + fmt.Fprintf(os.Stderr, "index: refreshed stale cache %s\n", autoPath) + case index.CacheFullyRebuilt: + fmt.Fprintf(os.Stderr, "index: rebuilt cache %s after configuration or root change\n", autoPath) + default: if len(excludes) > 0 { fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, applying %d exclusion patterns post-load)\n", autoPath, age, len(excludes)) } else { - fmt.Fprintf(os.Stderr, "index: using cached %s (age %s, pass --no-cache for fresh)\n", autoPath, age) + fmt.Fprintf(os.Stderr, "index: using fresh cache %s (age %s)\n", autoPath, age) } - return idx.ExcludePaths(excludes), false, nil } - fmt.Fprintf(os.Stderr, "index: config changed since last build, rebuilding...\n") + return idx.ExcludePaths(excludes), false, nil } } } @@ -129,18 +133,6 @@ func loadOrBuildWithScope(cmd *cobra.Command, cachePath string, target string, n return idx, false, err } -func configHashesMatch(cached, current map[string]string) bool { - if len(cached) != len(current) { - return false - } - for k, v := range cached { - if current[k] != v { - return false - } - } - return true -} - func emitJSON(value any) error { encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") diff --git a/cmd/canopy/helpers_test.go b/cmd/canopy/helpers_test.go index 53478e0..d13f848 100644 --- a/cmd/canopy/helpers_test.go +++ b/cmd/canopy/helpers_test.go @@ -7,6 +7,9 @@ import ( "testing" "github.com/spf13/cobra" + + "m31labs.dev/canopy/pkg/index" + "m31labs.dev/canopy/pkg/model" ) func TestLoadOrBuildChangedUsesChangedOnlyFallback(t *testing.T) { @@ -75,3 +78,51 @@ func TestLoadOrBuildCheckDerivesChangedPathsFromBase(t *testing.T) { t.Fatalf("check path = %q, want %q", got, want) } } + +func TestLoadOrBuildRefreshesAutoCacheWithoutConfig(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "main.go") + if err := os.WriteFile(sourcePath, []byte("package sample\n\nfunc Cached() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile(main.go) failed: %v", err) + } + builder, err := index.NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err) + } + cached, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + cachePath := filepath.Join(root, ".canopy", "index.json") + if err := index.Save(cachePath, cached); err != nil { + t.Fatalf("Save returned error: %v", err) + } + if cached.ConfigHashes != nil { + t.Fatalf("ConfigHashes = %#v, want nil for this regression", cached.ConfigHashes) + } + + if err := os.WriteFile(sourcePath, []byte("package sample\n\nfunc RefreshedWithLongerName() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile edited main.go failed: %v", err) + } + loaded, err := loadOrBuild(&cobra.Command{}, "", root, false) + if err != nil { + t.Fatalf("loadOrBuild returned error: %v", err) + } + if !helperIndexHasSymbol(loaded, "RefreshedWithLongerName") { + t.Fatal("auto-discovered cache did not refresh the edited source") + } + if helperIndexHasSymbol(loaded, "Cached") { + t.Fatal("auto-discovered cache retained the stale symbol") + } +} + +func helperIndexHasSymbol(idx *model.Index, name string) bool { + for _, file := range idx.Files { + for _, symbol := range file.Symbols { + if symbol.Name == name { + return true + } + } + } + return false +} diff --git a/cmd/canopy/index_validate.go b/cmd/canopy/index_validate.go index cbf05b9..5d9b442 100644 --- a/cmd/canopy/index_validate.go +++ b/cmd/canopy/index_validate.go @@ -1,23 +1,27 @@ package main import ( + "context" + "errors" "fmt" - "os" - "path/filepath" "github.com/spf13/cobra" "m31labs.dev/canopy/pkg/index" + "m31labs.dev/canopy/pkg/model" ) type validateReport struct { - Total int `json:"total"` - OK int `json:"ok"` - Stale int `json:"stale"` - Missing int `json:"missing"` - ParseErrors int `json:"parse_errors"` - StaleFiles []string `json:"stale_files,omitempty"` + Total int `json:"total"` + OK int `json:"ok"` + Stale int `json:"stale"` + Missing int `json:"missing"` + New int `json:"new"` + ParseErrors int `json:"parse_errors"` + RootMismatch bool `json:"root_mismatch,omitempty"` + StaleFiles []string `json:"stale_files,omitempty"` MissingFiles []string `json:"missing_files,omitempty"` + NewFiles []string `json:"new_files,omitempty"` } func newValidateCmd() *cobra.Command { @@ -26,7 +30,7 @@ func newValidateCmd() *cobra.Command { cmd := &cobra.Command{ Use: "validate [path]", - Short: "Check index integrity and detect stale or missing files", + Short: "Check index integrity and detect stale, missing, or new files", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if cachePath == "" { @@ -43,55 +47,28 @@ func newValidateCmd() *cobra.Command { root = args[0] } - report := validateReport{ - Total: len(idx.Files), - ParseErrors: len(idx.Errors), - } - - for _, f := range idx.Files { - absPath := f.Path - if !filepath.IsAbs(absPath) { - absPath = filepath.Join(root, absPath) - } - - info, err := os.Stat(absPath) - if err != nil { - report.Missing++ - report.MissingFiles = append(report.MissingFiles, f.Path) - continue - } - - if info.ModTime().After(idx.GeneratedAt) { - report.Stale++ - report.StaleFiles = append(report.StaleFiles, f.Path) - continue - } - - report.OK++ + report, err := buildValidateReport(cmd.Context(), root, idx) + if err != nil { + return err } if jsonOutput { - return emitJSON(report) - } - - fmt.Printf("validate: total=%d ok=%d stale=%d missing=%d parse_errors=%d\n", - report.Total, report.OK, report.Stale, report.Missing, report.ParseErrors) - - if len(report.MissingFiles) > 0 { - fmt.Println("missing:") - for _, p := range report.MissingFiles { - fmt.Printf(" %s\n", p) - } - } - if len(report.StaleFiles) > 0 { - fmt.Println("stale:") - for _, p := range report.StaleFiles { - fmt.Printf(" %s\n", p) + if err := emitJSON(report); err != nil { + return err } + } else { + fmt.Printf("validate: total=%d ok=%d stale=%d missing=%d new=%d parse_errors=%d root_mismatch=%t\n", + report.Total, report.OK, report.Stale, report.Missing, report.New, report.ParseErrors, report.RootMismatch) + printValidateFiles("missing", report.MissingFiles) + printValidateFiles("stale", report.StaleFiles) + printValidateFiles("new", report.NewFiles) } - if report.Stale > 0 || report.Missing > 0 { - os.Exit(2) + if report.RootMismatch || report.Stale > 0 || report.Missing > 0 || report.New > 0 { + return exitCodeError{ + code: 2, + err: errors.New("index source set is stale"), + } } return nil }, @@ -101,3 +78,51 @@ func newValidateCmd() *cobra.Command { cmd.Flags().BoolVar(&jsonOutput, "json", false, "emit JSON output") return cmd } + +func buildValidateReport(ctx context.Context, root string, idx *model.Index) (validateReport, error) { + builder, err := index.NewBuilderWithWorkspaceIgnores(root) + if err != nil { + return validateReport{}, err + } + freshness, err := builder.CheckFreshness(ctx, root, idx) + if err != nil { + return validateReport{}, err + } + + report := validateReport{ + Total: len(idx.Files), + OK: len(idx.Files), + Stale: len(freshness.StaleFiles), + Missing: len(freshness.MissingFiles), + New: len(freshness.NewFiles), + ParseErrors: len(idx.Errors), + RootMismatch: freshness.RootMismatch, + StaleFiles: freshness.StaleFiles, + MissingFiles: freshness.MissingFiles, + NewFiles: freshness.NewFiles, + } + if freshness.RootMismatch { + report.OK = 0 + return report, nil + } + indexedPaths := make(map[string]struct{}, len(idx.Files)) + for _, file := range idx.Files { + indexedPaths[file.Path] = struct{}{} + } + for _, path := range append(freshness.StaleFiles, freshness.MissingFiles...) { + if _, ok := indexedPaths[path]; ok { + report.OK-- + } + } + return report, nil +} + +func printValidateFiles(label string, paths []string) { + if len(paths) == 0 { + return + } + fmt.Printf("%s:\n", label) + for _, path := range paths { + fmt.Printf(" %s\n", path) + } +} diff --git a/cmd/canopy/index_validate_test.go b/cmd/canopy/index_validate_test.go new file mode 100644 index 0000000..77d5442 --- /dev/null +++ b/cmd/canopy/index_validate_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "m31labs.dev/canopy/pkg/index" + "m31labs.dev/canopy/pkg/model" +) + +func TestBuildValidateReportDetectsSourceChanges(t *testing.T) { + tests := []struct { + name string + change func(t *testing.T, root string) + wantOK int + wantStale int + wantMissing int + wantNew int + wantFiles []string + }{ + { + name: "edited indexed file", + change: func(t *testing.T, root string) { + t.Helper() + writeValidateSource(t, filepath.Join(root, "indexed.go"), "package sample\n\nfunc Changed() {}\n") + }, + wantStale: 1, + wantFiles: []string{"indexed.go"}, + }, + { + name: "missing indexed file", + change: func(t *testing.T, root string) { + t.Helper() + if err := os.Remove(filepath.Join(root, "indexed.go")); err != nil { + t.Fatalf("Remove(indexed.go) failed: %v", err) + } + }, + wantMissing: 1, + wantFiles: []string{"indexed.go"}, + }, + { + name: "new indexable file", + change: func(t *testing.T, root string) { + t.Helper() + writeValidateSource(t, filepath.Join(root, "added.go"), "package sample\n\nfunc Added() {}\n") + }, + wantOK: 1, + wantNew: 1, + wantFiles: []string{"added.go"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + indexedPath := filepath.Join(root, "indexed.go") + writeValidateSource(t, indexedPath, "package sample\n\nfunc Original() {}\n") + idx := validateTestIndex(t, root, indexedPath) + + tc.change(t, root) + + report, err := buildValidateReport(context.Background(), root, idx) + if err != nil { + t.Fatalf("buildValidateReport returned error: %v", err) + } + if report.Total != 1 { + t.Fatalf("Total = %d, want 1", report.Total) + } + if report.OK != tc.wantOK { + t.Fatalf("OK = %d, want %d", report.OK, tc.wantOK) + } + if report.Stale != tc.wantStale { + t.Fatalf("Stale = %d, want %d", report.Stale, tc.wantStale) + } + if report.Missing != tc.wantMissing { + t.Fatalf("Missing = %d, want %d", report.Missing, tc.wantMissing) + } + if report.New != tc.wantNew { + t.Fatalf("New = %d, want %d", report.New, tc.wantNew) + } + + var gotFiles []string + switch { + case tc.wantStale > 0: + gotFiles = report.StaleFiles + case tc.wantMissing > 0: + gotFiles = report.MissingFiles + case tc.wantNew > 0: + gotFiles = report.NewFiles + } + if !reflect.DeepEqual(gotFiles, tc.wantFiles) { + t.Fatalf("reported files = %#v, want %#v", gotFiles, tc.wantFiles) + } + }) + } +} + +func TestValidateCommandJSONReturnsStaleStatus(t *testing.T) { + root := t.TempDir() + indexedPath := filepath.Join(root, "indexed.go") + writeValidateSource(t, indexedPath, "package sample\n\nfunc Original() {}\n") + idx := validateTestIndex(t, root, indexedPath) + cachePath := filepath.Join(root, ".canopy", "index.json") + if err := index.Save(cachePath, idx); err != nil { + t.Fatalf("index.Save failed: %v", err) + } + writeValidateSource(t, indexedPath, "package sample\n\nfunc Changed() {}\n") + + output, err := captureValidateStdout(t, func() error { + cmd := newValidateCmd() + cmd.SilenceErrors = true + cmd.SilenceUsage = true + cmd.SetArgs([]string{root, "--cache", cachePath, "--json"}) + return cmd.Execute() + }) + + var exitErr interface{ ExitCode() int } + if !errors.As(err, &exitErr) { + t.Fatalf("Execute error = %v, want an exit code error", err) + } + if exitErr.ExitCode() != 2 { + t.Fatalf("exit code = %d, want 2", exitErr.ExitCode()) + } + + var report validateReport + if err := json.Unmarshal(output, &report); err != nil { + t.Fatalf("JSON output is invalid: %v\n%s", err, output) + } + if report.Stale != 1 || !reflect.DeepEqual(report.StaleFiles, []string{"indexed.go"}) { + t.Fatalf("JSON report = %+v, want indexed.go marked stale", report) + } +} + +func validateTestIndex(t *testing.T, root, indexedPath string) *model.Index { + t.Helper() + info, err := os.Stat(indexedPath) + if err != nil { + t.Fatalf("Stat(%s) failed: %v", indexedPath, err) + } + return &model.Index{ + Root: root, + GeneratedAt: time.Now(), + Files: []model.FileSummary{{ + Path: "indexed.go", + Language: "go", + SizeBytes: info.Size(), + ModTimeUnixNano: info.ModTime().UnixNano(), + }}, + } +} + +func writeValidateSource(t *testing.T, path, source string) { + t.Helper() + if err := os.WriteFile(path, []byte(source), 0o644); err != nil { + t.Fatalf("WriteFile(%s) failed: %v", path, err) + } +} + +func captureValidateStdout(t *testing.T, run func() error) ([]byte, error) { + t.Helper() + original := os.Stdout + readPipe, writePipe, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe failed: %v", err) + } + os.Stdout = writePipe + defer func() { + os.Stdout = original + }() + + runErr := run() + if err := writePipe.Close(); err != nil { + t.Fatalf("stdout writer close failed: %v", err) + } + output, err := io.ReadAll(readPipe) + if err != nil { + t.Fatalf("stdout read failed: %v", err) + } + if err := readPipe.Close(); err != nil { + t.Fatalf("stdout reader close failed: %v", err) + } + return output, runErr +} diff --git a/go.mod b/go.mod index 4c29ddc..169ccea 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25 require ( github.com/fsnotify/fsnotify v1.9.0 - github.com/odvcencio/gotreesitter v0.47.0 + github.com/odvcencio/gotreesitter v0.47.1-0.20260728141838-b8f61b592346 github.com/spf13/cobra v1.10.2 ) diff --git a/go.sum b/go.sum index 5c21c74..b9c1b85 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/odvcencio/gotreesitter v0.47.0 h1:M4n0d9JPqHUGeZubI8Rzd21cxKqCwA33ULSqoFFznGc= github.com/odvcencio/gotreesitter v0.47.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo= +github.com/odvcencio/gotreesitter v0.47.1-0.20260728141838-b8f61b592346 h1:cnNh3SRJYYC+m//sJYxdt9HngfoYecFKzc6jv3MwaiE= +github.com/odvcencio/gotreesitter v0.47.1-0.20260728141838-b8f61b592346/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= diff --git a/internal/mcp/helpers.go b/internal/mcp/helpers.go index a6e7286..5aea8c6 100644 --- a/internal/mcp/helpers.go +++ b/internal/mcp/helpers.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "fmt" "os" "path/filepath" @@ -19,25 +20,7 @@ func (s *Service) loadOrBuild(cachePath string, target string) (*model.Index, er if strings.TrimSpace(target) == "" { target = s.defaultRoot } - // Auto-discover cached index - autoPath := filepath.Join(target, ".canopy", "index.json") - if _, err := os.Stat(autoPath); err == nil { - if idx, loadErr := index.Load(autoPath); loadErr == nil { - if idx.ConfigHashes == nil { - return idx, nil // old cache — use it - } - current, hashErr := index.ComputeConfigHashes(target) - if hashErr == nil && configHashesMatch(idx.ConfigHashes, current) { - return idx, nil - } - // Config changed — fall through to rebuild. - } - } - builder, err := index.NewBuilderWithWorkspaceIgnores(target) - if err != nil { - return nil, err - } - return builder.BuildPath(target) + return s.loadOrBuildAuto(target) } func (s *Service) loadIndexFromSource(pathArg, cacheArg string) (*model.Index, error) { @@ -50,17 +33,19 @@ func (s *Service) loadIndexFromSource(pathArg, cacheArg string) (*model.Index, e if target == "" { target = s.defaultRoot } + return s.loadOrBuildAuto(target) +} + +func (s *Service) loadOrBuildAuto(target string) (*model.Index, error) { autoPath := filepath.Join(target, ".canopy", "index.json") if _, err := os.Stat(autoPath); err == nil { if idx, loadErr := index.Load(autoPath); loadErr == nil { - if idx.ConfigHashes == nil { - return idx, nil // old cache — use it + builder, buildErr := index.NewBuilderWithWorkspaceIgnores(target) + if buildErr != nil { + return nil, buildErr } - current, hashErr := index.ComputeConfigHashes(target) - if hashErr == nil && configHashesMatch(idx.ConfigHashes, current) { - return idx, nil - } - // Config changed — fall through to rebuild. + fresh, _, refreshErr := builder.EnsureFreshCache(context.Background(), target, autoPath, idx) + return fresh, refreshErr } } builder, err := index.NewBuilderWithWorkspaceIgnores(target) @@ -220,15 +205,3 @@ func isEntrypointDefinition(definition xref.Definition) bool { func isTestSourceFile(path string) bool { return strings.HasSuffix(strings.ToLower(strings.TrimSpace(path)), "_test.go") } - -func configHashesMatch(cached, current map[string]string) bool { - if len(cached) != len(current) { - return false - } - for k, v := range cached { - if current[k] != v { - return false - } - } - return true -} diff --git a/internal/mcp/helpers_freshness_test.go b/internal/mcp/helpers_freshness_test.go new file mode 100644 index 0000000..8a0022a --- /dev/null +++ b/internal/mcp/helpers_freshness_test.go @@ -0,0 +1,126 @@ +package mcp + +import ( + "os" + "path/filepath" + "testing" + + "m31labs.dev/canopy/pkg/index" + "m31labs.dev/canopy/pkg/model" +) + +type mcpIndexLoader func(service *Service, root, cachePath string) (*model.Index, error) + +func TestAutoDiscoveredCacheRefreshesAfterSourceEdit(t *testing.T) { + forEachMCPIndexLoader(t, func(t *testing.T, load mcpIndexLoader) { + root := t.TempDir() + writeMCPFreshnessSource(t, root, "main.go", "package sample\n\nfunc CachedVersion() {}\n") + saveMCPFreshnessCache(t, root, filepath.Join(root, ".canopy", "index.json")) + + writeMCPFreshnessSource(t, root, "main.go", "package sample\n\nfunc EditedVersionWithLongerName() {}\n") + + loaded, err := load(NewService(root, ""), root, "") + if err != nil { + t.Fatalf("load auto-discovered cache: %v", err) + } + requireMCPIndexSymbol(t, loaded, "EditedVersionWithLongerName", true) + requireMCPIndexSymbol(t, loaded, "CachedVersion", false) + }) +} + +func TestAutoDiscoveredCacheRefreshesAfterSourceFileAdded(t *testing.T) { + forEachMCPIndexLoader(t, func(t *testing.T, load mcpIndexLoader) { + root := t.TempDir() + writeMCPFreshnessSource(t, root, "main.go", "package sample\n\nfunc ExistingVersion() {}\n") + saveMCPFreshnessCache(t, root, filepath.Join(root, ".canopy", "index.json")) + + writeMCPFreshnessSource(t, root, "added.go", "package sample\n\nfunc AddedVersion() {}\n") + + loaded, err := load(NewService(root, ""), root, "") + if err != nil { + t.Fatalf("load auto-discovered cache: %v", err) + } + requireMCPIndexSymbol(t, loaded, "ExistingVersion", true) + requireMCPIndexSymbol(t, loaded, "AddedVersion", true) + }) +} + +func TestExplicitCachePathRemainsSnapshot(t *testing.T) { + forEachMCPIndexLoader(t, func(t *testing.T, load mcpIndexLoader) { + testDir := t.TempDir() + root := filepath.Join(testDir, "repo") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatalf("create source root: %v", err) + } + writeMCPFreshnessSource(t, root, "main.go", "package sample\n\nfunc SnapshotVersion() {}\n") + cachePath := filepath.Join(testDir, "snapshots", "index.json") + saveMCPFreshnessCache(t, root, cachePath) + + writeMCPFreshnessSource(t, root, "main.go", "package sample\n\nfunc LiveVersionWithLongerName() {}\n") + writeMCPFreshnessSource(t, root, "added.go", "package sample\n\nfunc AddedAfterSnapshot() {}\n") + + loaded, err := load(NewService(root, cachePath), root, cachePath) + if err != nil { + t.Fatalf("load explicit cache: %v", err) + } + requireMCPIndexSymbol(t, loaded, "SnapshotVersion", true) + requireMCPIndexSymbol(t, loaded, "LiveVersionWithLongerName", false) + requireMCPIndexSymbol(t, loaded, "AddedAfterSnapshot", false) + }) +} + +func forEachMCPIndexLoader(t *testing.T, test func(*testing.T, mcpIndexLoader)) { + t.Helper() + loaders := map[string]mcpIndexLoader{ + "loadOrBuild": func(service *Service, root, cachePath string) (*model.Index, error) { + return service.loadOrBuild(cachePath, root) + }, + "loadIndexFromSource": func(service *Service, root, cachePath string) (*model.Index, error) { + return service.loadIndexFromSource(root, cachePath) + }, + } + for name, load := range loaders { + t.Run(name, func(t *testing.T) { + test(t, load) + }) + } +} + +func writeMCPFreshnessSource(t *testing.T, root, name, source string) { + t.Helper() + path := filepath.Join(root, name) + if err := os.WriteFile(path, []byte(source), 0o644); err != nil { + t.Fatalf("write source %s: %v", name, err) + } +} + +func saveMCPFreshnessCache(t *testing.T, root, cachePath string) { + t.Helper() + builder, err := index.NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("create index builder: %v", err) + } + idx, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("build index: %v", err) + } + if err := index.Save(cachePath, idx); err != nil { + t.Fatalf("save index: %v", err) + } +} + +func requireMCPIndexSymbol(t *testing.T, idx *model.Index, name string, want bool) { + t.Helper() + found := false + for _, file := range idx.Files { + for _, symbol := range file.Symbols { + if symbol.Name == name { + found = true + break + } + } + } + if found != want { + t.Fatalf("symbol %q presence = %t, want %t", name, found, want) + } +} diff --git a/pkg/index/cache_refresh.go b/pkg/index/cache_refresh.go new file mode 100644 index 0000000..8d56292 --- /dev/null +++ b/pkg/index/cache_refresh.go @@ -0,0 +1,79 @@ +package index + +import ( + "context" + "fmt" + "strings" + + "m31labs.dev/canopy/pkg/model" +) + +// CacheRefreshStatus describes how EnsureFreshCache used the cached index. +type CacheRefreshStatus string + +const ( + CacheUnchanged CacheRefreshStatus = "unchanged" + CacheIncrementallyRefreshed CacheRefreshStatus = "incrementally_refreshed" + CacheFullyRebuilt CacheRefreshStatus = "fully_rebuilt" +) + +// EnsureFreshCache validates an auto-discovered cache and refreshes it when required. +func (b *Builder) EnsureFreshCache( + ctx context.Context, + target string, + cachePath string, + cached *model.Index, +) (*model.Index, CacheRefreshStatus, error) { + if b == nil { + return nil, "", fmt.Errorf("index builder is nil") + } + if cached == nil { + return nil, "", fmt.Errorf("cached index is nil") + } + if strings.TrimSpace(cachePath) == "" { + return nil, "", fmt.Errorf("cache path is empty") + } + if ctx == nil { + ctx = context.Background() + } + + configChanged := !configHashesEqual(cached.ConfigHashes, b.configHashes) + var report FreshnessReport + if !configChanged { + var err error + report, err = b.CheckFreshness(ctx, target, cached) + if err != nil { + return nil, "", err + } + if report.IsFresh() { + return cached, CacheUnchanged, nil + } + } + + base := cached + status := CacheIncrementallyRefreshed + if configChanged || report.RootMismatch { + base = nil + status = CacheFullyRebuilt + } + refreshed, _, err := b.BuildPathIncremental(ctx, target, base) + if err != nil { + return nil, "", err + } + if err := Save(cachePath, refreshed); err != nil { + return nil, "", fmt.Errorf("save refreshed cache: %w", err) + } + return refreshed, status, nil +} + +func configHashesEqual(cached, current map[string]string) bool { + if len(cached) != len(current) { + return false + } + for name, hash := range cached { + if current[name] != hash { + return false + } + } + return true +} diff --git a/pkg/index/freshness.go b/pkg/index/freshness.go new file mode 100644 index 0000000..80f2da2 --- /dev/null +++ b/pkg/index/freshness.go @@ -0,0 +1,156 @@ +package index + +import ( + "context" + "fmt" + "io/fs" + "path/filepath" + "sort" + "strings" + "time" + + "m31labs.dev/canopy/pkg/model" +) + +// FreshnessReport describes source changes against a cached index. +type FreshnessReport struct { + RootMismatch bool `json:"root_mismatch,omitempty"` + StaleFiles []string `json:"stale_files,omitempty"` + MissingFiles []string `json:"missing_files,omitempty"` + NewFiles []string `json:"new_files,omitempty"` +} + +// IsFresh reports whether the cache still describes the current source tree. +func (r FreshnessReport) IsFresh() bool { + return !r.RootMismatch && + len(r.StaleFiles) == 0 && + len(r.MissingFiles) == 0 && + len(r.NewFiles) == 0 +} + +// CheckFreshness compares source metadata with a cached index. +// It does not read or parse unchanged source files. +func (b *Builder) CheckFreshness(ctx context.Context, target string, cached *model.Index) (FreshnessReport, error) { + var report FreshnessReport + if cached == nil { + return report, fmt.Errorf("cached index is nil") + } + if ctx == nil { + ctx = context.Background() + } + + root, info, err := resolveBuildTarget(target) + if err != nil { + return report, err + } + if !info.IsDir() { + return report, fmt.Errorf("freshness target is not a directory: %s", root) + } + if filepath.Clean(cached.Root) != root { + report.RootMismatch = true + return report, nil + } + + filesByPath := make(map[string]model.FileSummary, len(cached.Files)) + for _, file := range cached.Files { + filesByPath[filepath.ToSlash(file.Path)] = file + } + errorsByPath := make(map[string]struct{}, len(cached.Errors)) + for _, parseErr := range cached.Errors { + errorsByPath[filepath.ToSlash(parseErr.Path)] = struct{}{} + } + seenFiles := make(map[string]struct{}, len(filesByPath)) + seenErrors := make(map[string]struct{}, len(errorsByPath)) + readyByExtension := map[string]bool{} + skipDirs := DefaultSkipDirs() + + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if entry.IsDir() { + if path == root { + return nil + } + if skipDirs[entry.Name()] || shouldSkipIndexPath(root, path, true, b.ignore) { + return filepath.SkipDir + } + return nil + } + if shouldSkipIndexPath(root, path, false, b.ignore) { + return nil + } + + relPath, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + relPath = filepath.ToSlash(relPath) + + if previous, ok := filesByPath[relPath]; ok { + seenFiles[relPath] = struct{}{} + fileInfo, infoErr := entry.Info() + if infoErr != nil { + return infoErr + } + if indexedFileMetadataChanged(previous, fileInfo.Size(), fileInfo.ModTime(), cached.GeneratedAt) { + report.StaleFiles = append(report.StaleFiles, relPath) + } + return nil + } + if _, ok := errorsByPath[relPath]; ok { + seenErrors[relPath] = struct{}{} + fileInfo, infoErr := entry.Info() + if infoErr != nil { + return infoErr + } + if fileInfo.ModTime().After(cached.GeneratedAt) { + report.StaleFiles = append(report.StaleFiles, relPath) + } + return nil + } + + parser, ok := b.parserForPath(path) + if !ok { + return nil + } + extension := strings.ToLower(filepath.Ext(path)) + ready, checked := readyByExtension[extension] + if !checked { + ready = parserReadyForIndex(parser) + readyByExtension[extension] = ready + } + if ready { + report.NewFiles = append(report.NewFiles, relPath) + } + return nil + }) + if err != nil { + return report, err + } + + for path := range filesByPath { + if _, ok := seenFiles[path]; !ok { + report.MissingFiles = append(report.MissingFiles, path) + } + } + for path := range errorsByPath { + if _, ok := seenErrors[path]; !ok { + report.MissingFiles = append(report.MissingFiles, path) + } + } + sort.Strings(report.StaleFiles) + sort.Strings(report.MissingFiles) + sort.Strings(report.NewFiles) + return report, nil +} + +func indexedFileMetadataChanged(previous model.FileSummary, size int64, modTime, generatedAt time.Time) bool { + if previous.ModTimeUnixNano == 0 { + return modTime.After(generatedAt) + } + return previous.SizeBytes != size || previous.ModTimeUnixNano != modTime.UnixNano() +} diff --git a/pkg/index/freshness_test.go b/pkg/index/freshness_test.go new file mode 100644 index 0000000..34851a0 --- /dev/null +++ b/pkg/index/freshness_test.go @@ -0,0 +1,230 @@ +package index + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + "m31labs.dev/canopy/pkg/model" +) + +func TestCheckFreshnessDetectsSourceTreeChanges(t *testing.T) { + tests := []struct { + name string + edit func(t *testing.T, root string) + want FreshnessReport + }{ + { + name: "clean", + edit: func(*testing.T, string) {}, + want: FreshnessReport{}, + }, + { + name: "edited", + edit: func(t *testing.T, root string) { + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc EditedWithLongerName() {}\n") + }, + want: FreshnessReport{StaleFiles: []string{"main.go"}}, + }, + { + name: "missing", + edit: func(t *testing.T, root string) { + if err := os.Remove(filepath.Join(root, "main.go")); err != nil { + t.Fatalf("Remove(main.go) failed: %v", err) + } + }, + want: FreshnessReport{MissingFiles: []string{"main.go"}}, + }, + { + name: "new indexable", + edit: func(t *testing.T, root string) { + writeFreshnessSource(t, root, "added.go", "package sample\n\nfunc Added() {}\n") + }, + want: FreshnessReport{NewFiles: []string{"added.go"}}, + }, + { + name: "new unsupported", + edit: func(t *testing.T, root string) { + writeFreshnessSource(t, root, "notes.unknown", "not source\n") + }, + want: FreshnessReport{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc Original() {}\n") + builder := NewBuilder() + cached, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + + tc.edit(t, root) + report, err := builder.CheckFreshness(context.Background(), root, cached) + if err != nil { + t.Fatalf("CheckFreshness returned error: %v", err) + } + if !reflect.DeepEqual(report, tc.want) { + t.Fatalf("CheckFreshness report = %+v, want %+v", report, tc.want) + } + if report.IsFresh() != tc.want.IsFresh() { + t.Fatalf("IsFresh = %t, want %t", report.IsFresh(), tc.want.IsFresh()) + } + }) + } +} + +func TestCheckFreshnessIgnoresNewExcludedSource(t *testing.T) { + root := t.TempDir() + writeFreshnessSource(t, root, ".graftignore", "ignored.go\n") + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc Original() {}\n") + builder, err := NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err) + } + cached, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + + writeFreshnessSource(t, root, "ignored.go", "package sample\n\nfunc Ignored() {}\n") + report, err := builder.CheckFreshness(context.Background(), root, cached) + if err != nil { + t.Fatalf("CheckFreshness returned error: %v", err) + } + if !report.IsFresh() { + t.Fatalf("ignored source made cache stale: %+v", report) + } +} + +func TestEnsureFreshCacheRefreshesAndSaves(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "main.go") + cachePath := filepath.Join(root, ".canopy", "index.json") + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc Original() {}\n") + builder, err := NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err) + } + cached, err := builder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + if err := Save(cachePath, cached); err != nil { + t.Fatalf("Save returned error: %v", err) + } + + if err := os.WriteFile(sourcePath, []byte("package sample\n\nfunc RefreshedWithLongerName() {}\n"), 0o644); err != nil { + t.Fatalf("WriteFile(main.go) failed: %v", err) + } + refreshed, status, err := builder.EnsureFreshCache(context.Background(), root, cachePath, cached) + if err != nil { + t.Fatalf("EnsureFreshCache returned error: %v", err) + } + if status != CacheIncrementallyRefreshed { + t.Fatalf("status = %q, want %q", status, CacheIncrementallyRefreshed) + } + if !indexHasSymbol(refreshed, "RefreshedWithLongerName") { + t.Fatal("refreshed index does not contain the edited symbol") + } + + saved, err := Load(cachePath) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if !indexHasSymbol(saved, "RefreshedWithLongerName") { + t.Fatal("saved cache does not contain the edited symbol") + } +} + +func TestEnsureFreshCacheRebuildsAfterConfigChange(t *testing.T) { + root := t.TempDir() + cachePath := filepath.Join(root, ".canopy", "index.json") + writeFreshnessSource(t, root, "main.go", "package sample\n\nfunc Main() {}\n") + writeFreshnessSource(t, root, "ignored.go", "package sample\n\nfunc Ignored() {}\n") + initialBuilder, err := NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores returned error: %v", err) + } + cached, err := initialBuilder.BuildPath(root) + if err != nil { + t.Fatalf("BuildPath returned error: %v", err) + } + if err := Save(cachePath, cached); err != nil { + t.Fatalf("Save returned error: %v", err) + } + + writeFreshnessSource(t, root, ".graftignore", "ignored.go\n") + currentBuilder, err := NewBuilderWithWorkspaceIgnores(root) + if err != nil { + t.Fatalf("NewBuilderWithWorkspaceIgnores after config change returned error: %v", err) + } + refreshed, status, err := currentBuilder.EnsureFreshCache(context.Background(), root, cachePath, cached) + if err != nil { + t.Fatalf("EnsureFreshCache returned error: %v", err) + } + if status != CacheFullyRebuilt { + t.Fatalf("status = %q, want %q", status, CacheFullyRebuilt) + } + if refreshed.FileCount() != 1 || indexHasSymbol(refreshed, "Ignored") { + t.Fatalf("config rebuild retained ignored source: %+v", refreshed.Files) + } +} + +func writeFreshnessSource(t *testing.T, root, name, source string) { + t.Helper() + if err := os.WriteFile(filepath.Join(root, name), []byte(source), 0o644); err != nil { + t.Fatalf("WriteFile(%s) failed: %v", name, err) + } +} + +func indexHasSymbol(idx *model.Index, name string) bool { + for _, file := range idx.Files { + for _, symbol := range file.Symbols { + if symbol.Name == name { + return true + } + } + } + return false +} + +func BenchmarkCheckFreshnessWarm1000(b *testing.B) { + root := b.TempDir() + files := make([]model.FileSummary, 0, 1000) + for i := 0; i < 1000; i++ { + name := fmt.Sprintf("file_%04d.go", i) + path := filepath.Join(root, name) + if err := os.WriteFile(path, []byte("package sample\n"), 0o644); err != nil { + b.Fatalf("WriteFile(%s) failed: %v", name, err) + } + info, err := os.Stat(path) + if err != nil { + b.Fatalf("Stat(%s) failed: %v", name, err) + } + files = append(files, model.FileSummary{ + Path: name, + Language: "go", + SizeBytes: info.Size(), + ModTimeUnixNano: info.ModTime().UnixNano(), + }) + } + cached := &model.Index{Root: root, Files: files} + builder := NewBuilder() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + report, err := builder.CheckFreshness(context.Background(), root, cached) + if err != nil { + b.Fatalf("CheckFreshness returned error: %v", err) + } + if !report.IsFresh() { + b.Fatalf("cache became stale: %+v", report) + } + } +}