From 42b5dc3c33c9aeb9503d857f5494ff8df0695906 Mon Sep 17 00:00:00 2001 From: Tym Rabchuk Date: Fri, 3 Apr 2026 01:02:05 -0400 Subject: [PATCH 1/3] add tests, test-gen command, review fixes, README, CI pipeline - 21 unit tests for lib/ and runners/ (db, metric, state, mock runner) - New test-gen command: generate tests, run, fix failures iteratively - Fix review findings: dynamic default branch detection, --agents whitespace trimming, status cost error handling, session ID validation - README: document Go CLI harness with all 9 commands and examples - GitHub Actions CI: build + vet + test + format check on push/PR --- .github/workflows/ci.yml | 35 ++++++++ README.md | 55 ++++++++++++ src/cmd/dispatch.go | 7 +- src/cmd/resume.go | 4 + src/cmd/review.go | 7 +- src/cmd/status.go | 8 +- src/cmd/testgen.go | 66 +++++++++++++++ src/lib/db_test.go | 168 +++++++++++++++++++++++++++++++++++++ src/lib/git.go | 24 +++++- src/lib/metric_test.go | 40 +++++++++ src/lib/state_test.go | 98 ++++++++++++++++++++++ src/loops/testgen.go | 131 +++++++++++++++++++++++++++++ src/runners/runner_test.go | 93 ++++++++++++++++++++ 13 files changed, 730 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 src/cmd/testgen.go create mode 100644 src/lib/db_test.go create mode 100644 src/lib/metric_test.go create mode 100644 src/lib/state_test.go create mode 100644 src/loops/testgen.go create mode 100644 src/runners/runner_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4a3d2ff --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: src/go.mod + + - name: Build + run: go build ./... + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... -v -race -count=1 + + - name: Check formatting + run: | + gofmt -l . + test -z "$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1) diff --git a/README.md b/README.md index 872769a..7908c08 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,61 @@ Set automatically in each multi-agent command: --- +## Go CLI Harness + +Deterministic orchestration binary — the machine controls the loop, Claude is the body. + +```bash +cd src && make build +./bin/devkit --help +``` + +### Commands + +| Command | Description | +|---|---| +| `devkit improve` | Metric-gated iteration loop — one Claude invocation per iteration | +| `devkit feature` | Plan, implement, test, lint — commits only after tests pass | +| `devkit bugfix` | Diagnose, fix, verify — reverts if tests break | +| `devkit refactor` | Analyze, transform, verify — reverts if behavior changes | +| `devkit test-gen` | Generate tests, run, fix failures — iterates until green | +| `devkit review` | Parallel multi-agent code review (Claude + Codex + Gemini) | +| `devkit dispatch` | Send any task to multiple agents, compare outputs | +| `devkit status` | Show all sessions, costs, iteration history | +| `devkit resume` | Pick up a crashed or paused session | + +### What it does that plugins can't + +- **Exact iteration counts** — Go binary owns the loop, not the LLM +- **Crash recovery** — SQLite state + handoff files survive crashes +- **Hard budget caps** — stops spawning at your dollar limit +- **CI/CD integration** — runs headless, no conversation needed +- **True parallel dispatch** — goroutines, not sequential prompts + +### Examples + +```bash +# Run 50 improvement iterations overnight, stop at $20 +devkit improve --metric "npm test" --iterations 50 --budget 20.00 + +# Implement a feature with test verification +devkit feature "add JWT auth" --target src/auth/ --test "npm test" + +# Fix a bug with automated verification +devkit bugfix "login 500 on plus sign emails" --test "go test ./..." + +# Generate tests for a module +devkit test-gen src/parser/ --test "go test ./..." + +# Resume a crashed session +devkit resume abc123def456 + +# Check what happened +devkit status +``` + +--- + ## Prerequisites **Required:** Claude Code (you're already here) diff --git a/src/cmd/dispatch.go b/src/cmd/dispatch.go index 3cba0df..60be42e 100644 --- a/src/cmd/dispatch.go +++ b/src/cmd/dispatch.go @@ -23,7 +23,12 @@ var dispatchCmd = &cobra.Command{ var agents []string if agentList != "" { - agents = strings.Split(agentList, ",") + for _, a := range strings.Split(agentList, ",") { + a = strings.TrimSpace(a) + if a != "" { + agents = append(agents, a) + } + } } available := runners.DetectRunners() diff --git a/src/cmd/resume.go b/src/cmd/resume.go index 258a9f4..0fe5346 100644 --- a/src/cmd/resume.go +++ b/src/cmd/resume.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "regexp" "github.com/5uck1ess/devkit/lib" "github.com/5uck1ess/devkit/loops" @@ -16,6 +17,9 @@ var resumeCmd = &cobra.Command{ Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { sessionID := args[0] + if !regexp.MustCompile(`^[a-f0-9]{12}$`).MatchString(sessionID) { + return fmt.Errorf("invalid session ID %q — expected 12 hex characters (e.g., a1b2c3d4e5f6)", sessionID) + } session, err := db.GetSession(sessionID) if err != nil { diff --git a/src/cmd/review.go b/src/cmd/review.go index 9f57b5c..b8737a6 100644 --- a/src/cmd/review.go +++ b/src/cmd/review.go @@ -26,7 +26,12 @@ var reviewCmd = &cobra.Command{ var agents []string if agentList != "" { - agents = strings.Split(agentList, ",") + for _, a := range strings.Split(agentList, ",") { + a = strings.TrimSpace(a) + if a != "" { + agents = append(agents, a) + } + } } available := runners.DetectRunners() diff --git a/src/cmd/status.go b/src/cmd/status.go index c7f0ea7..a054b17 100644 --- a/src/cmd/status.go +++ b/src/cmd/status.go @@ -39,9 +39,13 @@ func showAllSessions() error { fmt.Printf("%-14s %-10s %-10s %-8s %s\n", "-------", "--------", "------", "----", "-------") for _, s := range sessions { - cost, _ := db.SessionTotalCost(s.ID) + cost, err := db.SessionTotalCost(s.ID) + costStr := fmt.Sprintf("$%.4f", cost) + if err != nil { + costStr = "unknown" + } age := formatAge(s.CreatedAt) - fmt.Printf("%-14s %-10s %-10s $%-7.4f %s\n", s.ID, s.Workflow, s.Status, cost, age) + fmt.Printf("%-14s %-10s %-10s %-8s %s\n", s.ID, s.Workflow, s.Status, costStr, age) } return nil } diff --git a/src/cmd/testgen.go b/src/cmd/testgen.go new file mode 100644 index 0000000..32cf0f3 --- /dev/null +++ b/src/cmd/testgen.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/5uck1ess/devkit/lib" + "github.com/5uck1ess/devkit/loops" + "github.com/5uck1ess/devkit/runners" + "github.com/spf13/cobra" +) + +var testGenCmd = &cobra.Command{ + Use: "test-gen [target]", + Short: "Generate tests for target code, run them, fix failures", + Long: "Analyzes target code, generates comprehensive tests, runs them, and iterates until green.", + Example: ` devkit test-gen src/auth/ + devkit test-gen lib/parser.go --test "go test ./..." + devkit test-gen src/ --test "npm test" --budget 5.00`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + testCmd, _ := cmd.Flags().GetString("test") + budget, _ := cmd.Flags().GetFloat64("budget") + + dirty, err := (&lib.Git{Dir: repoRoot}).HasUncommittedChanges() + if err != nil { + return fmt.Errorf("check git status: %w", err) + } + if dirty { + return fmt.Errorf("working tree has uncommitted changes — commit or stash first") + } + + available := runners.DetectRunners() + runner := runners.FindRunner("claude", available) + if runner == nil { + return fmt.Errorf("claude CLI not found in PATH") + } + + result, err := loops.RunTestGen(cmd.Context(), db, runner, &lib.Git{Dir: repoRoot}, loops.TestGenConfig{ + Target: strings.Join(args, " "), + TestCmd: testCmd, + RepoRoot: repoRoot, + BudgetUSD: budget, + }) + if err != nil { + return err + } + + var totalCost float64 + for _, s := range result.Steps { + totalCost += s.CostUSD + } + fmt.Printf("\n=== Test Generation Complete ===\n") + fmt.Printf("Session: %s\n", result.Session.ID) + fmt.Printf("Steps: %d\n", len(result.Steps)) + fmt.Printf("Cost: $%.4f\n", totalCost) + fmt.Printf("\nRun `devkit status %s` for details.\n", result.Session.ID) + return nil + }, +} + +func init() { + rootCmd.AddCommand(testGenCmd) + testGenCmd.Flags().String("test", "", "Test command to run generated tests") + testGenCmd.Flags().Float64("budget", 0, "Maximum spend in USD (0 = unlimited)") +} diff --git a/src/lib/db_test.go b/src/lib/db_test.go new file mode 100644 index 0000000..46724a6 --- /dev/null +++ b/src/lib/db_test.go @@ -0,0 +1,168 @@ +package lib + +import ( + "os" + "path/filepath" + "testing" +) + +func tempDB(t *testing.T) *DB { + t.Helper() + dir := t.TempDir() + db, err := OpenDB(filepath.Join(dir, ".devkit", "devkit.db")) + if err != nil { + t.Fatalf("open db: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestCreateAndGetSession(t *testing.T) { + db := tempDB(t) + + s := &Session{ + ID: "abc123def456", + Workflow: "improve", + Target: "src/", + Metric: "go test ./...", + Status: "running", + } + if err := db.CreateSession(s); err != nil { + t.Fatalf("create: %v", err) + } + + got, err := db.GetSession("abc123def456") + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Workflow != "improve" { + t.Errorf("workflow = %q, want improve", got.Workflow) + } + if got.Status != "running" { + t.Errorf("status = %q, want running", got.Status) + } +} + +func TestGetSessionNotFound(t *testing.T) { + db := tempDB(t) + _, err := db.GetSession("nonexistent") + if err == nil { + t.Fatal("expected error for nonexistent session") + } +} + +func TestUpdateSessionStatus(t *testing.T) { + db := tempDB(t) + s := &Session{ID: "test12345678", Workflow: "improve", Status: "running"} + db.CreateSession(s) + + if err := db.UpdateSessionStatus("test12345678", "done"); err != nil { + t.Fatalf("update: %v", err) + } + + got, _ := db.GetSession("test12345678") + if got.Status != "done" { + t.Errorf("status = %q, want done", got.Status) + } +} + +func TestCreateAndGetSteps(t *testing.T) { + db := tempDB(t) + db.CreateSession(&Session{ID: "sess12345678", Workflow: "improve", Status: "running"}) + + step := &Step{SessionID: "sess12345678", Iteration: 1, Status: "running", AgentName: "claude"} + if err := db.CreateStep(step); err != nil { + t.Fatalf("create step: %v", err) + } + if step.ID == 0 { + t.Error("step ID should be set after create") + } + + step.Status = "kept" + step.Kept = true + step.MetricExitCode = 0 + step.CostUSD = 0.05 + if err := db.UpdateStep(step); err != nil { + t.Fatalf("update step: %v", err) + } + + steps, err := db.GetSteps("sess12345678") + if err != nil { + t.Fatalf("get steps: %v", err) + } + if len(steps) != 1 { + t.Fatalf("got %d steps, want 1", len(steps)) + } + if !steps[0].Kept { + t.Error("step should be kept") + } +} + +func TestSessionTotalCost(t *testing.T) { + db := tempDB(t) + db.CreateSession(&Session{ID: "cost12345678", Workflow: "improve", Status: "running"}) + + for i := 1; i <= 3; i++ { + s := &Step{SessionID: "cost12345678", Iteration: i, Status: "done", AgentName: "claude"} + db.CreateStep(s) + s.CostUSD = 0.10 + db.UpdateStep(s) + } + + cost, err := db.SessionTotalCost("cost12345678") + if err != nil { + t.Fatalf("total cost: %v", err) + } + if cost < 0.29 || cost > 0.31 { + t.Errorf("cost = %f, want ~0.30", cost) + } +} + +func TestListSessions(t *testing.T) { + db := tempDB(t) + db.CreateSession(&Session{ID: "list12345678", Workflow: "improve", Status: "done"}) + db.CreateSession(&Session{ID: "list87654321", Workflow: "review", Status: "done"}) + + sessions, err := db.ListSessions() + if err != nil { + t.Fatalf("list: %v", err) + } + if len(sessions) != 2 { + t.Errorf("got %d sessions, want 2", len(sessions)) + } +} + +func TestLastIteration(t *testing.T) { + db := tempDB(t) + db.CreateSession(&Session{ID: "iter12345678", Workflow: "improve", Status: "running"}) + + iter, _ := db.LastIteration("iter12345678") + if iter != 0 { + t.Errorf("empty session should have last iter 0, got %d", iter) + } + + for i := 1; i <= 5; i++ { + s := &Step{SessionID: "iter12345678", Iteration: i, Status: "done", AgentName: "claude"} + db.CreateStep(s) + } + + iter, _ = db.LastIteration("iter12345678") + if iter != 5 { + t.Errorf("last iter = %d, want 5", iter) + } +} + +func TestDBDirectoryPermissions(t *testing.T) { + dir := t.TempDir() + dbDir := filepath.Join(dir, ".devkit") + OpenDB(filepath.Join(dbDir, "devkit.db")) + + info, err := os.Stat(dbDir) + if err != nil { + t.Fatalf("stat: %v", err) + } + perm := info.Mode().Perm() + if perm != 0o700 { + t.Errorf("directory permissions = %o, want 700", perm) + } +} diff --git a/src/lib/git.go b/src/lib/git.go index f740fca..882a915 100644 --- a/src/lib/git.go +++ b/src/lib/git.go @@ -67,10 +67,30 @@ func (g *Git) DiffStat() (string, error) { return g.run("diff", "--cached", "--stat") } +func (g *Git) DefaultBranch() string { + // Try to detect from remote + if ref, err := g.run("symbolic-ref", "refs/remotes/origin/HEAD"); err == nil { + parts := strings.Split(ref, "/") + return parts[len(parts)-1] + } + // Fallback: try main, then master + if _, err := g.run("rev-parse", "--verify", "main"); err == nil { + return "main" + } + if _, err := g.run("rev-parse", "--verify", "master"); err == nil { + return "master" + } + return "main" +} + func (g *Git) DiffFromMain() (string, error) { - diff, err := g.run("diff", "main...HEAD") + base := g.DefaultBranch() + diff, err := g.run("diff", base+"...HEAD") + if err != nil { + // fallback to cached + unstaged + diff, err = g.run("diff", "HEAD") + } if err != nil { - // fallback to cached diff diff, err = g.run("diff", "--cached") } return diff, err diff --git a/src/lib/metric_test.go b/src/lib/metric_test.go new file mode 100644 index 0000000..4b45f5e --- /dev/null +++ b/src/lib/metric_test.go @@ -0,0 +1,40 @@ +package lib + +import ( + "context" + "testing" +) + +func TestRunMetricSuccess(t *testing.T) { + result := RunMetric(context.Background(), "echo hello", t.TempDir()) + if result.ExitCode != 0 { + t.Errorf("exit code = %d, want 0", result.ExitCode) + } + if result.Output == "" { + t.Error("output should not be empty") + } +} + +func TestRunMetricFailure(t *testing.T) { + result := RunMetric(context.Background(), "exit 1", t.TempDir()) + if result.ExitCode != 1 { + t.Errorf("exit code = %d, want 1", result.ExitCode) + } +} + +func TestRunMetricTruncation(t *testing.T) { + // Generate output larger than 4096 bytes + result := RunMetric(context.Background(), "head -c 5000 /dev/zero | tr '\\0' 'a'", t.TempDir()) + if len(result.Output) > 4200 { + t.Errorf("output should be truncated, got %d bytes", len(result.Output)) + } +} + +func TestRunMetricCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := RunMetric(ctx, "sleep 10", t.TempDir()) + if result.ExitCode == 0 { + t.Error("cancelled command should have non-zero exit code") + } +} diff --git a/src/lib/state_test.go b/src/lib/state_test.go new file mode 100644 index 0000000..af3cbed --- /dev/null +++ b/src/lib/state_test.go @@ -0,0 +1,98 @@ +package lib + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNewSessionID(t *testing.T) { + id := NewSessionID() + if len(id) != 12 { + t.Errorf("session ID length = %d, want 12", len(id)) + } + // Should be hex + for _, c := range id { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + t.Errorf("session ID contains non-hex character: %c", c) + } + } +} + +func TestNewSessionIDUnique(t *testing.T) { + ids := make(map[string]bool) + for i := 0; i < 100; i++ { + id := NewSessionID() + if ids[id] { + t.Fatalf("duplicate session ID: %s", id) + } + ids[id] = true + } +} + +func TestEnsureSessionDir(t *testing.T) { + root := t.TempDir() + id := "test12345678" + + if err := EnsureSessionDir(root, id); err != nil { + t.Fatalf("ensure dir: %v", err) + } + + dir := SessionDir(root, id) + if _, err := os.Stat(dir); os.IsNotExist(err) { + t.Error("session directory was not created") + } +} + +func TestWriteHandoff(t *testing.T) { + root := t.TempDir() + session := &Session{ + ID: "hand12345678", + Workflow: "improve", + Target: "src/", + Objective: "fix all tests", + Metric: "go test ./...", + MaxIterations: 10, + BudgetUSD: 5.00, + } + + steps := []Step{ + {Iteration: 1, Kept: true, MetricExitCode: 0, CostUSD: 0.05, ChangeSummary: "fixed auth"}, + {Iteration: 2, Kept: false, MetricExitCode: 1, CostUSD: 0.03, ChangeSummary: "broke tests"}, + } + + baseline := MetricResult{ExitCode: 1, Output: "3 tests failed"} + + if err := WriteHandoff(root, session, steps, baseline); err != nil { + t.Fatalf("write handoff: %v", err) + } + + path := HandoffPath(root, session.ID) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read handoff: %v", err) + } + + content := string(data) + if !strings.Contains(content, "Iteration: 3 of 10") { + t.Error("handoff should show next iteration as 3") + } + if !strings.Contains(content, "fix all tests") { + t.Error("handoff should contain objective") + } + if !strings.Contains(content, "fixed auth") { + t.Error("handoff should contain iteration history") + } + if !strings.Contains(content, "$4.92") { + t.Errorf("handoff should show remaining budget, got:\n%s", content) + } +} + +func TestHandoffPath(t *testing.T) { + path := HandoffPath("/repo", "abc123def456") + expected := filepath.Join("/repo", ".devkit", "sessions", "abc123def456", "handoff.md") + if path != expected { + t.Errorf("path = %s, want %s", path, expected) + } +} diff --git a/src/loops/testgen.go b/src/loops/testgen.go new file mode 100644 index 0000000..67a8be0 --- /dev/null +++ b/src/loops/testgen.go @@ -0,0 +1,131 @@ +package loops + +import ( + "context" + "fmt" + + "github.com/5uck1ess/devkit/lib" + "github.com/5uck1ess/devkit/runners" +) + +type TestGenConfig struct { + Target string + TestCmd string + RepoRoot string + BudgetUSD float64 +} + +type TestGenResult struct { + Session *lib.Session + Steps []lib.Step +} + +func RunTestGen(ctx context.Context, db *lib.DB, runner runners.Runner, git *lib.Git, cfg TestGenConfig) (*TestGenResult, error) { + session := &lib.Session{ + ID: lib.NewSessionID(), + Workflow: "test-gen", + Target: cfg.Target, + Metric: cfg.TestCmd, + Status: "running", + BudgetUSD: cfg.BudgetUSD, + } + if err := db.CreateSession(session); err != nil { + return nil, fmt.Errorf("create session: %w", err) + } + if err := lib.EnsureSessionDir(cfg.RepoRoot, session.ID); err != nil { + return nil, fmt.Errorf("create session directory: %w", err) + } + + branchName := fmt.Sprintf("test-gen/%s", session.ID) + if err := git.CreateBranch(branchName); err != nil { + return nil, fmt.Errorf("create branch: %w", err) + } + fmt.Printf("Test-gen session %s on branch %s\n\n", session.ID, branchName) + + var spentUSD float64 + checkBudget := func() bool { + return cfg.BudgetUSD > 0 && spentUSD >= cfg.BudgetUSD + } + opts := runners.RunOpts{ + WorkDir: cfg.RepoRoot, + AllowedTools: "Bash,Read,Edit,Write,Grep,Glob", + MaxTurns: 30, + } + + // Step 1: Analyze target and generate tests + fmt.Println("--- Step 1: Generate Tests ---") + genStep := &lib.Step{SessionID: session.ID, Iteration: 1, Status: "running", AgentName: runner.Name()} + db.CreateStep(genStep) + + genResult, err := runner.Run(ctx, fmt.Sprintf( + `Analyze the code at %s and generate a comprehensive test suite. + +1. Detect the language, test framework, and existing test patterns. +2. Identify all public functions, methods, and API endpoints. +3. Write tests covering: happy paths, edge cases, error conditions, boundary values. +4. Use the project's existing test framework and conventions. +5. Place tests in the project's standard test location. + +Write actual test code — no placeholders or TODOs.`, cfg.Target), opts) + if err != nil { + genStep.Status = "failed" + genStep.ChangeSummary = err.Error() + db.UpdateStep(genStep) + db.UpdateSessionStatus(session.ID, "failed") + return nil, fmt.Errorf("generate step failed: %w", err) + } + spentUSD += genResult.CostUSD + git.CommitAll(fmt.Sprintf("test-gen(%s): generate tests", session.ID)) + genStep.Status = "kept" + genStep.Kept = true + genStep.CostUSD = genResult.CostUSD + genStep.ChangeSummary = truncate(genResult.Output, 200) + db.UpdateStep(genStep) + fmt.Printf(" Tests generated ($%.4f)\n\n", genResult.CostUSD) + + // Step 2: Run tests and fix failures (up to 5 attempts) + if cfg.TestCmd != "" && !checkBudget() { + fmt.Println("--- Step 2: Run & Fix ---") + for attempt := 1; attempt <= 5; attempt++ { + if ctx.Err() != nil || checkBudget() { + break + } + testMetric := lib.RunMetric(ctx, cfg.TestCmd, cfg.RepoRoot) + if testMetric.ExitCode == 0 { + fmt.Printf(" All tests passing (attempt %d)\n", attempt) + break + } + + fmt.Printf(" Tests failing (attempt %d), fixing...\n", attempt) + fixStep := &lib.Step{SessionID: session.ID, Iteration: 1 + attempt, Status: "running", AgentName: runner.Name()} + db.CreateStep(fixStep) + + fixResult, err := runner.Run(ctx, fmt.Sprintf( + `The generated tests are failing. Fix them so they pass. +Determine if the bug is in the test or the implementation. +If the test expectation is wrong, fix the test. If the code has a bug, fix the code. + +Test command: %s +Test output: +%s`, cfg.TestCmd, testMetric.Output), opts) + if err != nil { + fixStep.Status = "failed" + fixStep.ChangeSummary = err.Error() + db.UpdateStep(fixStep) + continue + } + spentUSD += fixResult.CostUSD + git.CommitAll(fmt.Sprintf("test-gen(%s): fix tests attempt %d", session.ID, attempt)) + fixStep.Status = "kept" + fixStep.Kept = true + fixStep.CostUSD = fixResult.CostUSD + db.UpdateStep(fixStep) + } + } + + db.UpdateSessionStatus(session.ID, "done") + allSteps, _ := db.GetSteps(session.ID) + lib.WriteReport(cfg.RepoRoot, session, allSteps, "completed") + + return &TestGenResult{Session: session, Steps: allSteps}, nil +} diff --git a/src/runners/runner_test.go b/src/runners/runner_test.go new file mode 100644 index 0000000..3c544f1 --- /dev/null +++ b/src/runners/runner_test.go @@ -0,0 +1,93 @@ +package runners + +import ( + "context" + "testing" +) + +// MockRunner for testing loops without real CLI calls +type MockRunner struct { + name string + responses []RunResult + errors []error + callIdx int +} + +func NewMockRunner(name string, responses []RunResult, errors []error) *MockRunner { + return &MockRunner{name: name, responses: responses, errors: errors} +} + +func (m *MockRunner) Name() string { return m.name } +func (m *MockRunner) Available() bool { return true } + +func (m *MockRunner) Run(ctx context.Context, prompt string, opts RunOpts) (RunResult, error) { + idx := m.callIdx + m.callIdx++ + if idx >= len(m.responses) { + return RunResult{Output: "mock exhausted"}, nil + } + var err error + if idx < len(m.errors) { + err = m.errors[idx] + } + return m.responses[idx], err +} + +func (m *MockRunner) CallCount() int { return m.callIdx } + +func TestDetectRunners(t *testing.T) { + available := DetectRunners() + // At minimum claude should be available in dev environment + // but we can't guarantee it in CI, so just check it doesn't panic + if available == nil { + // nil is fine — means nothing is installed + } +} + +func TestFindRunner(t *testing.T) { + runners := []Runner{ + NewMockRunner("claude", nil, nil), + NewMockRunner("codex", nil, nil), + } + + found := FindRunner("claude", runners) + if found == nil || found.Name() != "claude" { + t.Error("should find claude runner") + } + + notFound := FindRunner("gemini", runners) + if notFound != nil { + t.Error("should not find gemini runner") + } +} + +func TestTruncStr(t *testing.T) { + if TruncStr("short", 10) != "short" { + t.Error("short string should not be truncated") + } + result := TruncStr("this is a long string", 10) + if result != "this is a ..." { + t.Errorf("got %q, want %q", result, "this is a ...") + } +} + +func TestMockRunner(t *testing.T) { + mock := NewMockRunner("test", []RunResult{ + {Output: "first", CostUSD: 0.01}, + {Output: "second", CostUSD: 0.02}, + }, nil) + + r1, _ := mock.Run(context.Background(), "p1", RunOpts{}) + if r1.Output != "first" { + t.Errorf("first call output = %q", r1.Output) + } + + r2, _ := mock.Run(context.Background(), "p2", RunOpts{}) + if r2.Output != "second" { + t.Errorf("second call output = %q", r2.Output) + } + + if mock.CallCount() != 2 { + t.Errorf("call count = %d, want 2", mock.CallCount()) + } +} From 08f550ed5deef0d4b6fe0aba193dcc454c341192 Mon Sep 17 00:00:00 2001 From: Tym Rabchuk Date: Fri, 3 Apr 2026 01:18:57 -0400 Subject: [PATCH 2/3] fix all tri-review findings: session ID fallback, testgen status, test quality Critical: - NewSessionID fallback now produces fixed-length 12-char hex (was PID-length) Warnings: - testgen: track test pass/fail, mark session "failed" if tests never pass - testgen: handle CommitAll and GetSteps errors - resume: compile regex at package level, normalize input to lowercase - DefaultBranch: use TrimPrefix for slash-safe branch name parsing - DB tests: check all setup errors via mustCreateSession/mustCreateStep helpers - DB permissions test: handle OpenDB error, close DB, use mask comparison Suggestions: - TruncStr: use rune-based truncation (UTF-8 safe) - Metric truncation test: portable printf instead of /dev/zero - TestDetectRunners: assert runner names and Available() consistency - CI gofmt: run once instead of three times --- .github/workflows/ci.yml | 8 +++-- src/cmd/resume.go | 7 ++-- src/lib/db_test.go | 67 +++++++++++++++++++++++++------------- src/lib/git.go | 3 +- src/lib/metric_test.go | 9 ++--- src/lib/state.go | 5 +-- src/loops/testgen.go | 23 ++++++++++--- src/runners/runner.go | 7 ++-- src/runners/runner_test.go | 11 ++++--- 9 files changed, 94 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a3d2ff..9ee5953 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,5 +31,9 @@ jobs: - name: Check formatting run: | - gofmt -l . - test -z "$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1) + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "Files not formatted:" + echo "$unformatted" + exit 1 + fi diff --git a/src/cmd/resume.go b/src/cmd/resume.go index 0fe5346..846c113 100644 --- a/src/cmd/resume.go +++ b/src/cmd/resume.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "regexp" + "strings" "github.com/5uck1ess/devkit/lib" "github.com/5uck1ess/devkit/loops" @@ -10,14 +11,16 @@ import ( "github.com/spf13/cobra" ) +var sessionIDPattern = regexp.MustCompile(`^[a-f0-9]{12}$`) + var resumeCmd = &cobra.Command{ Use: "resume ", Short: "Resume a paused or crashed session", Long: "Picks up an improve session from where it left off, using the SQLite state and handoff file.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - sessionID := args[0] - if !regexp.MustCompile(`^[a-f0-9]{12}$`).MatchString(sessionID) { + sessionID := strings.ToLower(args[0]) + if !sessionIDPattern.MatchString(sessionID) { return fmt.Errorf("invalid session ID %q — expected 12 hex characters (e.g., a1b2c3d4e5f6)", sessionID) } diff --git a/src/lib/db_test.go b/src/lib/db_test.go index 46724a6..62f162b 100644 --- a/src/lib/db_test.go +++ b/src/lib/db_test.go @@ -17,6 +17,20 @@ func tempDB(t *testing.T) *DB { return db } +func mustCreateSession(t *testing.T, db *DB, s *Session) { + t.Helper() + if err := db.CreateSession(s); err != nil { + t.Fatalf("setup: create session: %v", err) + } +} + +func mustCreateStep(t *testing.T, db *DB, s *Step) { + t.Helper() + if err := db.CreateStep(s); err != nil { + t.Fatalf("setup: create step: %v", err) + } +} + func TestCreateAndGetSession(t *testing.T) { db := tempDB(t) @@ -27,9 +41,7 @@ func TestCreateAndGetSession(t *testing.T) { Metric: "go test ./...", Status: "running", } - if err := db.CreateSession(s); err != nil { - t.Fatalf("create: %v", err) - } + mustCreateSession(t, db, s) got, err := db.GetSession("abc123def456") if err != nil { @@ -53,14 +65,16 @@ func TestGetSessionNotFound(t *testing.T) { func TestUpdateSessionStatus(t *testing.T) { db := tempDB(t) - s := &Session{ID: "test12345678", Workflow: "improve", Status: "running"} - db.CreateSession(s) + mustCreateSession(t, db, &Session{ID: "test12345678", Workflow: "improve", Status: "running"}) if err := db.UpdateSessionStatus("test12345678", "done"); err != nil { t.Fatalf("update: %v", err) } - got, _ := db.GetSession("test12345678") + got, err := db.GetSession("test12345678") + if err != nil { + t.Fatalf("get: %v", err) + } if got.Status != "done" { t.Errorf("status = %q, want done", got.Status) } @@ -68,12 +82,10 @@ func TestUpdateSessionStatus(t *testing.T) { func TestCreateAndGetSteps(t *testing.T) { db := tempDB(t) - db.CreateSession(&Session{ID: "sess12345678", Workflow: "improve", Status: "running"}) + mustCreateSession(t, db, &Session{ID: "sess12345678", Workflow: "improve", Status: "running"}) step := &Step{SessionID: "sess12345678", Iteration: 1, Status: "running", AgentName: "claude"} - if err := db.CreateStep(step); err != nil { - t.Fatalf("create step: %v", err) - } + mustCreateStep(t, db, step) if step.ID == 0 { t.Error("step ID should be set after create") } @@ -100,13 +112,15 @@ func TestCreateAndGetSteps(t *testing.T) { func TestSessionTotalCost(t *testing.T) { db := tempDB(t) - db.CreateSession(&Session{ID: "cost12345678", Workflow: "improve", Status: "running"}) + mustCreateSession(t, db, &Session{ID: "cost12345678", Workflow: "improve", Status: "running"}) for i := 1; i <= 3; i++ { s := &Step{SessionID: "cost12345678", Iteration: i, Status: "done", AgentName: "claude"} - db.CreateStep(s) + mustCreateStep(t, db, s) s.CostUSD = 0.10 - db.UpdateStep(s) + if err := db.UpdateStep(s); err != nil { + t.Fatalf("update step %d: %v", i, err) + } } cost, err := db.SessionTotalCost("cost12345678") @@ -120,8 +134,8 @@ func TestSessionTotalCost(t *testing.T) { func TestListSessions(t *testing.T) { db := tempDB(t) - db.CreateSession(&Session{ID: "list12345678", Workflow: "improve", Status: "done"}) - db.CreateSession(&Session{ID: "list87654321", Workflow: "review", Status: "done"}) + mustCreateSession(t, db, &Session{ID: "list12345678", Workflow: "improve", Status: "done"}) + mustCreateSession(t, db, &Session{ID: "list87654321", Workflow: "review", Status: "done"}) sessions, err := db.ListSessions() if err != nil { @@ -134,19 +148,24 @@ func TestListSessions(t *testing.T) { func TestLastIteration(t *testing.T) { db := tempDB(t) - db.CreateSession(&Session{ID: "iter12345678", Workflow: "improve", Status: "running"}) + mustCreateSession(t, db, &Session{ID: "iter12345678", Workflow: "improve", Status: "running"}) - iter, _ := db.LastIteration("iter12345678") + iter, err := db.LastIteration("iter12345678") + if err != nil { + t.Fatalf("last iteration: %v", err) + } if iter != 0 { t.Errorf("empty session should have last iter 0, got %d", iter) } for i := 1; i <= 5; i++ { - s := &Step{SessionID: "iter12345678", Iteration: i, Status: "done", AgentName: "claude"} - db.CreateStep(s) + mustCreateStep(t, db, &Step{SessionID: "iter12345678", Iteration: i, Status: "done", AgentName: "claude"}) } - iter, _ = db.LastIteration("iter12345678") + iter, err = db.LastIteration("iter12345678") + if err != nil { + t.Fatalf("last iteration: %v", err) + } if iter != 5 { t.Errorf("last iter = %d, want 5", iter) } @@ -155,14 +174,18 @@ func TestLastIteration(t *testing.T) { func TestDBDirectoryPermissions(t *testing.T) { dir := t.TempDir() dbDir := filepath.Join(dir, ".devkit") - OpenDB(filepath.Join(dbDir, "devkit.db")) + db, err := OpenDB(filepath.Join(dbDir, "devkit.db")) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() info, err := os.Stat(dbDir) if err != nil { t.Fatalf("stat: %v", err) } perm := info.Mode().Perm() - if perm != 0o700 { + if perm&0o777 != 0o700 { t.Errorf("directory permissions = %o, want 700", perm) } } diff --git a/src/lib/git.go b/src/lib/git.go index 882a915..0afc839 100644 --- a/src/lib/git.go +++ b/src/lib/git.go @@ -70,8 +70,7 @@ func (g *Git) DiffStat() (string, error) { func (g *Git) DefaultBranch() string { // Try to detect from remote if ref, err := g.run("symbolic-ref", "refs/remotes/origin/HEAD"); err == nil { - parts := strings.Split(ref, "/") - return parts[len(parts)-1] + return strings.TrimPrefix(ref, "refs/remotes/origin/") } // Fallback: try main, then master if _, err := g.run("rev-parse", "--verify", "main"); err == nil { diff --git a/src/lib/metric_test.go b/src/lib/metric_test.go index 4b45f5e..49b5a04 100644 --- a/src/lib/metric_test.go +++ b/src/lib/metric_test.go @@ -23,10 +23,11 @@ func TestRunMetricFailure(t *testing.T) { } func TestRunMetricTruncation(t *testing.T) { - // Generate output larger than 4096 bytes - result := RunMetric(context.Background(), "head -c 5000 /dev/zero | tr '\\0' 'a'", t.TempDir()) - if len(result.Output) > 4200 { - t.Errorf("output should be truncated, got %d bytes", len(result.Output)) + // Generate output larger than 4096 bytes using portable printf + result := RunMetric(context.Background(), "printf '%5000s' ' ' | tr ' ' 'a'", t.TempDir()) + maxExpected := 4096 + len("\n... (truncated)") + if len(result.Output) > maxExpected { + t.Errorf("output should be truncated to ~%d, got %d bytes", maxExpected, len(result.Output)) } } diff --git a/src/lib/state.go b/src/lib/state.go index 9ada4e1..74df0d0 100644 --- a/src/lib/state.go +++ b/src/lib/state.go @@ -7,13 +7,14 @@ import ( "os" "path/filepath" "strings" + "time" ) func NewSessionID() string { b := make([]byte, 6) if _, err := rand.Read(b); err != nil { - // Fallback to timestamp-based ID if crypto/rand fails - return fmt.Sprintf("%x", os.Getpid()) + // Fallback: fixed-length 12-char hex from nanosecond timestamp + return fmt.Sprintf("%012x", time.Now().UnixNano())[:12] } return hex.EncodeToString(b) } diff --git a/src/loops/testgen.go b/src/loops/testgen.go index 67a8be0..577ed33 100644 --- a/src/loops/testgen.go +++ b/src/loops/testgen.go @@ -75,7 +75,9 @@ Write actual test code — no placeholders or TODOs.`, cfg.Target), opts) return nil, fmt.Errorf("generate step failed: %w", err) } spentUSD += genResult.CostUSD - git.CommitAll(fmt.Sprintf("test-gen(%s): generate tests", session.ID)) + if err := git.CommitAll(fmt.Sprintf("test-gen(%s): generate tests", session.ID)); err != nil { + fmt.Printf(" Warning: commit failed: %s\n", err) + } genStep.Status = "kept" genStep.Kept = true genStep.CostUSD = genResult.CostUSD @@ -84,6 +86,7 @@ Write actual test code — no placeholders or TODOs.`, cfg.Target), opts) fmt.Printf(" Tests generated ($%.4f)\n\n", genResult.CostUSD) // Step 2: Run tests and fix failures (up to 5 attempts) + testsPass := cfg.TestCmd == "" if cfg.TestCmd != "" && !checkBudget() { fmt.Println("--- Step 2: Run & Fix ---") for attempt := 1; attempt <= 5; attempt++ { @@ -93,6 +96,7 @@ Write actual test code — no placeholders or TODOs.`, cfg.Target), opts) testMetric := lib.RunMetric(ctx, cfg.TestCmd, cfg.RepoRoot) if testMetric.ExitCode == 0 { fmt.Printf(" All tests passing (attempt %d)\n", attempt) + testsPass = true break } @@ -115,7 +119,9 @@ Test output: continue } spentUSD += fixResult.CostUSD - git.CommitAll(fmt.Sprintf("test-gen(%s): fix tests attempt %d", session.ID, attempt)) + if err := git.CommitAll(fmt.Sprintf("test-gen(%s): fix tests attempt %d", session.ID, attempt)); err != nil { + fmt.Printf(" Warning: commit failed: %s\n", err) + } fixStep.Status = "kept" fixStep.Kept = true fixStep.CostUSD = fixResult.CostUSD @@ -123,9 +129,16 @@ Test output: } } - db.UpdateSessionStatus(session.ID, "done") - allSteps, _ := db.GetSteps(session.ID) - lib.WriteReport(cfg.RepoRoot, session, allSteps, "completed") + status := "done" + if !testsPass { + status = "failed" + } + db.UpdateSessionStatus(session.ID, status) + allSteps, err := db.GetSteps(session.ID) + if err != nil { + fmt.Printf(" Warning: failed to get steps for report: %s\n", err) + } + lib.WriteReport(cfg.RepoRoot, session, allSteps, status) return &TestGenResult{Session: session, Steps: allSteps}, nil } diff --git a/src/runners/runner.go b/src/runners/runner.go index 82198e8..63678ff 100644 --- a/src/runners/runner.go +++ b/src/runners/runner.go @@ -49,10 +49,11 @@ func FindRunner(name string, runners []Runner) Runner { return nil } -// TruncStr truncates a string to n bytes with "..." suffix. +// TruncStr truncates a string to n runes with "..." suffix. func TruncStr(s string, n int) string { - if len(s) <= n { + runes := []rune(s) + if len(runes) <= n { return s } - return s[:n] + "..." + return string(runes[:n]) + "..." } diff --git a/src/runners/runner_test.go b/src/runners/runner_test.go index 3c544f1..fb360d6 100644 --- a/src/runners/runner_test.go +++ b/src/runners/runner_test.go @@ -37,10 +37,13 @@ func (m *MockRunner) CallCount() int { return m.callIdx } func TestDetectRunners(t *testing.T) { available := DetectRunners() - // At minimum claude should be available in dev environment - // but we can't guarantee it in CI, so just check it doesn't panic - if available == nil { - // nil is fine — means nothing is installed + for _, r := range available { + if r.Name() == "" { + t.Error("runner has empty name") + } + if !r.Available() { + t.Errorf("runner %s reported as available but Available() returns false", r.Name()) + } } } From ad555740a0c44c584ad26f89c48900a44b440cf1 Mon Sep 17 00:00:00 2001 From: Tym Rabchuk Date: Fri, 3 Apr 2026 01:21:06 -0400 Subject: [PATCH 3/3] add harness detection to all tri-* commands, install target improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 5 tri-* commands now check if the devkit binary is in PATH and delegate to it when found — avoids output truncation, gets SQLite session tracking, and full stdout capture for free. Falls back to plugin-based orchestration if harness not installed. Makefile: add link target, install prints GOPATH/bin path guidance. --- commands/tri-debug.md | 12 ++++++++++++ commands/tri-dispatch.md | 12 ++++++++++++ commands/tri-review.md | 14 ++++++++++++++ commands/tri-security.md | 12 ++++++++++++ commands/tri-test-gen.md | 12 ++++++++++++ src/Makefile | 8 +++++++- 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/commands/tri-debug.md b/commands/tri-debug.md index 2a6ea61..9a2cae0 100644 --- a/commands/tri-debug.md +++ b/commands/tri-debug.md @@ -7,6 +7,18 @@ description: Multi-agent debugging — send a bug report to available agents (Cl Send a bug description to all available agents in parallel, get independent root-cause analyses, and consolidate into a recommended fix. +## Step 0: Harness Detection + +```bash +if command -v devkit >/dev/null 2>&1; then + echo "Go harness detected — delegating to devkit dispatch for full output capture." + devkit dispatch {prompt with bug context} + exit 0 +fi +``` + +If the `devkit` binary is in PATH, delegate entirely to it. Only fall through to plugin-based steps if the harness is not installed. + ## Step 1: Gather Context Collect from the user: diff --git a/commands/tri-dispatch.md b/commands/tri-dispatch.md index f2b7c31..0162c92 100644 --- a/commands/tri-dispatch.md +++ b/commands/tri-dispatch.md @@ -7,6 +7,18 @@ description: Dispatch a task to all three agents (Claude, Codex, Gemini) in para Send the same task to Claude, Codex, and Gemini in parallel. Compare outputs. +## Step 0: Harness Detection + +```bash +if command -v devkit >/dev/null 2>&1; then + echo "Go harness detected — delegating to devkit dispatch for full output capture." + devkit dispatch {prompt} + exit 0 +fi +``` + +If the `devkit` binary is in PATH, delegate entirely to it. Only fall through to plugin-based steps if the harness is not installed. + ## When to use - Comparing approaches to a problem diff --git a/commands/tri-review.md b/commands/tri-review.md index 1272d62..fd68d9d 100644 --- a/commands/tri-review.md +++ b/commands/tri-review.md @@ -7,6 +7,20 @@ description: Triple-agent PR/code review. Claude runs as native background agent Run the same code review across three AI agents in parallel and consolidate results. +## Step 0: Harness Detection + +```bash +if command -v devkit >/dev/null 2>&1; then + echo "Go harness detected — delegating to devkit review for full output capture." + devkit review {prompt or default} + # The harness handles parallel dispatch, full stdout capture (no truncation), + # SQLite session tracking, and consolidated output. Skip all steps below. + exit 0 +fi +``` + +If the `devkit` binary is in PATH, delegate entirely to it. The harness avoids output truncation, captures full agent responses, and tracks sessions in SQLite. Only fall through to the plugin-based steps below if the harness is not installed. + ## Step 1: Gather Context ```bash diff --git a/commands/tri-security.md b/commands/tri-security.md index 8d5d37c..0547a9d 100644 --- a/commands/tri-security.md +++ b/commands/tri-security.md @@ -7,6 +7,18 @@ description: Multi-agent security audit — independent security reviews from av Independent security reviews from all available agents, consolidated into a severity-ranked report. +## Step 0: Harness Detection + +```bash +if command -v devkit >/dev/null 2>&1; then + echo "Go harness detected — delegating to devkit review --security for full output capture." + devkit review --security {prompt or default} + exit 0 +fi +``` + +If the `devkit` binary is in PATH, delegate entirely to it. Only fall through to plugin-based steps if the harness is not installed. + ## Step 1: Gather Scope Determine what to audit: diff --git a/commands/tri-test-gen.md b/commands/tri-test-gen.md index d72fe4a..998f374 100644 --- a/commands/tri-test-gen.md +++ b/commands/tri-test-gen.md @@ -7,6 +7,18 @@ description: Multi-agent test generation — each available agent generates test Generate tests from all available agents in parallel, then merge the best tests into a comprehensive suite. +## Step 0: Harness Detection + +```bash +if command -v devkit >/dev/null 2>&1; then + echo "Go harness detected — delegating to devkit test-gen for full output capture." + devkit test-gen {target} --test {test_command} + exit 0 +fi +``` + +If the `devkit` binary is in PATH, delegate entirely to it. Only fall through to plugin-based steps if the harness is not installed. + ## Step 1: Analyze Target Read the target files and detect: diff --git a/src/Makefile b/src/Makefile index 25c0ba4..d6decde 100644 --- a/src/Makefile +++ b/src/Makefile @@ -3,7 +3,7 @@ VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev LDFLAGS := -s -w -X main.version=$(VERSION) GOFLAGS := -trimpath -.PHONY: build install clean test vet fmt check all +.PHONY: build install link clean test vet fmt check all all: check build @@ -12,6 +12,12 @@ build: install: go install $(GOFLAGS) -ldflags '$(LDFLAGS)' . + @echo "Installed to $$(go env GOPATH)/bin/$(BINARY)" + @echo "Ensure $$(go env GOPATH)/bin is in your PATH" + +link: build + ln -sf $(CURDIR)/bin/$(BINARY) /usr/local/bin/$(BINARY) + @echo "Linked bin/$(BINARY) → /usr/local/bin/$(BINARY)" clean: rm -rf bin/