From 2196e6079454afef07398aba4229bae57036c721 Mon Sep 17 00:00:00 2001 From: crossben Date: Fri, 25 Sep 2026 15:17:55 +0000 Subject: [PATCH 01/25] tui: strip terminal colour codes from agent replies; indent wrapped notes opencode colours its output. The live panel already cleaned it, but the final reply went to glamour raw, which drops ESC and leaves literal "[0m" in the transcript. Replies (and failing-check output in review) now go through the same cleaning. Long system notes also wrap with an indent instead of starting at column 0. Co-Authored-By: Claude Opus 5.5 --- internal/tui/chat.go | 9 +++++---- internal/tui/live.go | 10 ++++++++++ internal/tui/review.go | 2 +- internal/tui/tui_test.go | 28 ++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/internal/tui/chat.go b/internal/tui/chat.go index 7aa9a50..9bbcac8 100644 --- a/internal/tui/chat.go +++ b/internal/tui/chat.go @@ -152,12 +152,12 @@ func (m Model) onTurn(msg turnMsg) (tea.Model, tea.Cmd) { case msg.answered: run.result = "answered" run.agent = "" - m.messages = append(m.messages, chatLine{role: "agent", text: strings.TrimSpace(t.AgentText), agent: "orchestra"}) + m.messages = append(m.messages, chatLine{role: "agent", text: cleanText(t.AgentText), agent: "orchestra"}) m.logEvent("turn", "question answered") - ui.Notify("Orchestra", "Answered — "+firstLine(t.AgentText)) + ui.Notify("Orchestra", "Answered — "+firstLine(cleanText(t.AgentText))) case !t.HadChanges: run.result = "no changes" - resp := strings.TrimSpace(t.AgentText) + resp := cleanText(t.AgentText) if resp == "" { resp = "(the agent made no file changes)" } @@ -302,7 +302,8 @@ func sysLine(text string, w int) string { case strings.HasPrefix(text, "↺"), strings.HasPrefix(text, "●"): st = warnSty } - return lipgloss.NewStyle().Width(w).Render(" " + st.Render(text)) + // Padding (not a literal indent) keeps wrapped lines aligned under the first. + return lipgloss.NewStyle().Width(w).PaddingLeft(2).Render(st.Render(text)) } func (m Model) welcome() string { diff --git a/internal/tui/live.go b/internal/tui/live.go index 7f83e30..40ebf9d 100644 --- a/internal/tui/live.go +++ b/internal/tui/live.go @@ -210,6 +210,16 @@ func cleanLine(s string) string { return strings.ReplaceAll(s, "\t", " ") } +// cleanText applies cleanLine to every line of an agent's captured output, so +// colour codes from CLIs like opencode never reach the transcript as "[0m". +func cleanText(s string) string { + lines := strings.Split(s, "\n") + for i, l := range lines { + lines[i] = cleanLine(l) + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + // onRunMsg applies one worker message to the model. func (m Model) onRunMsg(msg tea.Msg) (tea.Model, tea.Cmd) { run := m.run diff --git a/internal/tui/review.go b/internal/tui/review.go index 441a329..b700447 100644 --- a/internal/tui/review.go +++ b/internal/tui/review.go @@ -250,7 +250,7 @@ func (r reviewer) failureTail() []string { return nil } var lines []string - for _, l := range strings.Split(strings.TrimRight(f.Output, "\n"), "\n") { + for _, l := range strings.Split(cleanText(f.Output), "\n") { if strings.TrimSpace(l) != "" { lines = append(lines, l) } diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index a259247..9f39a8e 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -10,6 +10,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" "github.com/crossben/orchestra-code/internal/agent" "github.com/crossben/orchestra-code/internal/config" @@ -524,3 +525,30 @@ func TestCtrlCDuringRunRevertsThenQuits(t *testing.T) { t.Fatal("partial change should be reverted before quitting") } } + +// opencode colours its output; the escape codes must not leak into the chat +// transcript as literal "[0m" (glamour drops ESC but keeps the rest). +func TestAgentReplyColourCodesStripped(t *testing.T) { + m := testModel() + m.cstate = chatRunning + m.run = &liveRun{task: "hello", start: time.Now(), agent: "opencode"} + nm, _ := m.onTurn(turnMsg{turn: engine.Turn{AgentText: "\x1b[0m\n\x1b[1m│ build · big-pickle \x1b[0m Hello!\r\n"}, agent: "opencode", started: true}) + // Real escape sequences are fine (styling); leftover "[0m" text is not. + v := ansi.Strip(nm.(Model).View()) + if strings.Contains(v, "[0m") || strings.Contains(v, "[1m") { + t.Fatalf("colour codes leaked into the transcript:\n%s", v) + } + if !strings.Contains(v, "Hello!") { + t.Fatalf("reply text missing:\n%s", v) + } +} + +// A long system note wraps with its continuation lines indented too. +func TestSysLineWrapKeepsIndent(t *testing.T) { + out := sysLine("↳ routed to opencode — classification failed (exec: \"claude\": executable file not found in $PATH); defaulting to implement", 60) + for i, l := range strings.Split(out, "\n") { + if !strings.HasPrefix(l, " ") { + t.Fatalf("line %d not indented: %q", i, l) + } + } +} From 37f34150b10b0b66b37aa05b18eac2ac3376477f Mon Sep 17 00:00:00 2001 From: crossben Date: Fri, 25 Sep 2026 15:27:33 +0000 Subject: [PATCH 02/25] tui: scroll long answers with the mouse wheel and arrow keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only pgup/pgdn scrolled the transcript; arrows went to the input and the program never enabled mouse reporting, so a long reply looked stuck at its end. - Mouse wheel scrolls the transcript and open diffs, and moves the selection in Changes/History (tea.WithMouseCellMotion). - ↑/↓ scroll the transcript while the input is empty. - Status bar shows "scrolled · N%" when not at the bottom. Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/dashboard.go | 4 ++- internal/tui/chat.go | 8 +++++ internal/tui/chrome.go | 4 ++- internal/tui/tui.go | 40 +++++++++++++++++++++++ internal/tui/tui_test.go | 65 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 2 deletions(-) diff --git a/cmd/orchestra/dashboard.go b/cmd/orchestra/dashboard.go index eb72d49..74f505f 100644 --- a/cmd/orchestra/dashboard.go +++ b/cmd/orchestra/dashboard.go @@ -62,7 +62,9 @@ func newDashboardCmd() *cobra.Command { Principles: config.PrinciplesText(cfg.Principles), DefaultAgent: cfg.DefaultAgent, }) - p := tea.NewProgram(model, tea.WithAltScreen(), tea.WithContext(cmd.Context())) + // Mouse reporting enables wheel scrolling; to select text for + // copying, most terminals need shift held while dragging. + p := tea.NewProgram(model, tea.WithAltScreen(), tea.WithMouseCellMotion(), tea.WithContext(cmd.Context())) _, err = p.Run() return err }, diff --git a/internal/tui/chat.go b/internal/tui/chat.go index 9bbcac8..79e01b8 100644 --- a/internal/tui/chat.go +++ b/internal/tui/chat.go @@ -89,6 +89,14 @@ func (m Model) updateChat(msg tea.KeyMsg) (tea.Model, tea.Cmd) { switch msg.String() { case "enter": return m.submitChat() + case "up", "down": + // With nothing typed, the arrows scroll the conversation; once there is + // text they move the cursor in the input as usual. + if strings.TrimSpace(m.ta.Value()) == "" { + var cmd tea.Cmd + m.vp, cmd = m.vp.Update(msg) + return m, cmd + } case "pgup", "pgdown": var cmd tea.Cmd m.vp, cmd = m.vp.Update(msg) diff --git a/internal/tui/chrome.go b/internal/tui/chrome.go index 385ef05..b59d4f8 100644 --- a/internal/tui/chrome.go +++ b/internal/tui/chrome.go @@ -71,6 +71,8 @@ func (m Model) statusBar() string { right = dimSty.Render(m.status) case m.cstate == chatRunning && m.run != nil: right = warnSty.Render(fmtElapsed(time.Since(m.run.start))) + case m.active == tabChat && m.cstate == chatIdle && !m.browsing && !m.vp.AtBottom(): + right = dimSty.Render(fmt.Sprintf("scrolled · %d%%", int(m.vp.ScrollPercent()*100))) } left := mode + " " + hints @@ -94,7 +96,7 @@ func (m Model) hints() []string { case chatReviewing: return append(m.rv.hints(), "tab switch") } - return []string{"enter send", "ctrl+j newline", "pgup/pgdn scroll", "tab switch", "ctrl+c quit"} + return []string{"enter send", "ctrl+j newline", "↑↓/wheel scroll", "tab switch", "ctrl+c quit"} case tabChanges, tabHistory: return []string{"↑↓ select", "enter open", "r refresh", "tab switch", "q quit"} case tabAgents: diff --git a/internal/tui/tui.go b/internal/tui/tui.go index a067d9d..f8b1eed 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -290,6 +290,8 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m.onRunMsg(msg) case tea.KeyMsg: return m.onKey(msg) + case tea.MouseMsg: + return m.onMouse(msg), nil } // Non-key messages (e.g. cursor blink) go to the input. if m.active == tabChat && m.cstate == chatIdle { @@ -363,6 +365,44 @@ func (m Model) onKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { return m, nil } +// wheelLines is how far one mouse-wheel notch scrolls. +const wheelLines = 3 + +// onMouse scrolls whatever is under focus with the wheel: the transcript, an +// open diff, or the selection in a list. +func (m Model) onMouse(msg tea.MouseMsg) Model { + if msg.Action != tea.MouseActionPress { + return m + } + var dir int + switch msg.Button { + case tea.MouseButtonWheelUp: + dir = -1 + case tea.MouseButtonWheelDown: + dir = 1 + default: + return m + } + scroll := func(vp *viewport.Model) { + if dir < 0 { + vp.LineUp(wheelLines) + } else { + vp.LineDown(wheelLines) + } + } + switch { + case m.browsing: + scroll(&m.browse.vp) + case m.active == tabChat && m.cstate == chatReviewing: + scroll(&m.rv.vp) + case m.active == tabChat: + scroll(&m.vp) + case m.active == tabChanges, m.active == tabHistory: + m.moveSel(dir) + } + return m +} + func (m Model) switchTab(t tab) Model { if t == m.active { return m diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index 9f39a8e..e31614a 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -552,3 +552,68 @@ func TestSysLineWrapKeepsIndent(t *testing.T) { } } } + +// longAnswer puts a many-screen reply in the transcript, scrolled to the end. +func longAnswer(t *testing.T) Model { + t.Helper() + m := testModelIn(".", nil, nil, 120, 30) + m.messages = []chatLine{{role: "you", text: "explain"}} + m.cstate = chatRunning + m.run = &liveRun{task: "explain", start: time.Now()} + var b strings.Builder + b.WriteString("FIRSTLINE of the answer\n\n") + for i := 0; i < 80; i++ { + b.WriteString("a paragraph of the explanation\n\n") + } + nm, _ := m.onTurn(turnMsg{turn: engine.Turn{AgentText: b.String()}, answered: true}) + m = nm.(Model) + if m.vp.AtTop() || strings.Contains(m.View(), "FIRSTLINE") { + t.Fatal("a long answer should open scrolled to its end") + } + return m +} + +func TestLongAnswerScrollsToTopWithArrows(t *testing.T) { + m := longAnswer(t) + for i := 0; i < 400 && !m.vp.AtTop(); i++ { + m = press(m, "up") + } + if !strings.Contains(m.View(), "FIRSTLINE") { + t.Fatal("↑ with an empty input should scroll back to the start") + } + if !strings.Contains(m.View(), "scrolled") { + t.Fatal("status bar should say the transcript is scrolled") + } + // With text typed, arrows belong to the input again. + m = press(m, "hi") + before := m.vp.YOffset + m = press(m, "down") + if m.vp.YOffset != before { + t.Fatal("arrows should not scroll while there is text in the input") + } +} + +func TestLongAnswerScrollsWithMouseWheel(t *testing.T) { + m := longAnswer(t) + wheel := func(m Model, b tea.MouseButton) Model { + nm, _ := m.Update(tea.MouseMsg{Action: tea.MouseActionPress, Button: b}) + return nm.(Model) + } + for i := 0; i < 200 && !m.vp.AtTop(); i++ { + m = wheel(m, tea.MouseButtonWheelUp) + } + if !strings.Contains(m.View(), "FIRSTLINE") { + t.Fatal("wheel up should scroll back to the start") + } + m = wheel(m, tea.MouseButtonWheelDown) + if m.vp.YOffset != wheelLines { + t.Fatalf("wheel down should scroll %d lines, offset=%d", wheelLines, m.vp.YOffset) + } + // In History the wheel moves the selection. + h := m.switchTab(tabHistory) + h.runs = []memory.Run{{Agent: "a", Prompt: "one"}, {Agent: "a", Prompt: "two"}} + h.reload() + if h = wheel(h, tea.MouseButtonWheelDown); h.histSel != 1 { + t.Fatalf("wheel should move the History selection, sel=%d", h.histSel) + } +} From dc01d472dea472f671627625a345e8ee6df2aa87 Mon Sep 17 00:00:00 2001 From: crossben Date: Fri, 25 Sep 2026 15:34:24 +0000 Subject: [PATCH 03/25] tui: show a resize hint in windows smaller than 60x12 Below the minimum the layout was clipped from the top (header and tabs disappeared). Frames now always fit the window, and tiny windows say what size is needed. Co-Authored-By: Claude Opus 5.5 --- internal/tui/chrome.go | 9 +++++++++ internal/tui/tui.go | 8 ++++++++ internal/tui/tui_test.go | 22 ++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/internal/tui/chrome.go b/internal/tui/chrome.go index b59d4f8..56206e3 100644 --- a/internal/tui/chrome.go +++ b/internal/tui/chrome.go @@ -140,6 +140,15 @@ func padLines(s string, n, w int) string { return strings.Join(lines, "\n") } +// tooSmall fills tiny windows with a resize hint, never exceeding them. +func (m Model) tooSmall() string { + w, h := max(m.width, 1), max(m.height, 1) + msg := titleSty.Render("⬡ ORCHESTRA") + "\n" + + dimSty.Render(fmt.Sprintf("window is %d×%d — make it at least %d×%d", m.width, m.height, minWidth, minHeight)) + "\n" + + dimSty.Render("ctrl+c quits") + return padLines(lipgloss.Place(w, h, lipgloss.Center, lipgloss.Center, msg), h, w) +} + // emptyState is a centred hint box for tabs with nothing to show yet. func emptyState(title, hint string, w, h int) string { b := box("", headSty.Render(title)+"\n"+dimSty.Render(hint), min(max(lipgloss.Width(hint)+6, 40), w), faint) diff --git a/internal/tui/tui.go b/internal/tui/tui.go index f8b1eed..b6a8438 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -493,10 +493,18 @@ func (m *Model) logEvent(kind, text string) { // bodyHeight is the space between the header (2 lines) and status bar (1). func (m Model) bodyHeight() int { return max(m.height-3, 5) } +// Smallest window the full layout fits in. +const minWidth, minHeight = 60, 12 + func (m Model) View() string { if !m.ready { return "loading…" } + // A frame taller than the window makes the terminal scroll on every + // redraw, stacking copies of the screen; below the minimum, say so instead. + if m.width < minWidth || m.height < minHeight { + return m.tooSmall() + } var body string switch { case m.browsing: diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index e31614a..0afe58e 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -617,3 +617,25 @@ func TestLongAnswerScrollsWithMouseWheel(t *testing.T) { t.Fatalf("wheel should move the History selection, sel=%d", h.histSel) } } + +// In any window, however small, a frame must fit exactly: taller frames make +// the terminal scroll on every redraw and stack copies of the screen. +func TestTinyWindowNeverOverflows(t *testing.T) { + for _, size := range [][2]int{{80, 2}, {80, 5}, {40, 20}, {59, 11}, {20, 3}, {1, 1}} { + m := testModelIn(".", nil, nil, size[0], size[1]) + for tb := tab(0); tb < numTabs; tb++ { + lines := strings.Split(m.switchTab(tb).View(), "\n") + if len(lines) > size[1] { + t.Fatalf("%dx%d tab %s: %d lines", size[0], size[1], tabNames[tb], len(lines)) + } + for _, l := range lines { + if lipgloss.Width(l) > size[0] { + t.Fatalf("%dx%d tab %s: line wider than window: %q", size[0], size[1], tabNames[tb], l) + } + } + } + } + if v := testModelIn(".", nil, nil, 80, 8).View(); !strings.Contains(v, "at least") { + t.Fatalf("small window should explain the minimum size:\n%s", v) + } +} From 1d8a17b3ea160af046e0e1ec107826599f9532ad Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 19:55:51 +0000 Subject: [PATCH 04/25] docs: design spec for git-free parallel do and benchmark Co-Authored-By: Claude Opus 5.5 --- .../2026-09-26-gitfree-parallel-design.md | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-26-gitfree-parallel-design.md diff --git a/docs/superpowers/specs/2026-09-26-gitfree-parallel-design.md b/docs/superpowers/specs/2026-09-26-gitfree-parallel-design.md new file mode 100644 index 0000000..37e4463 --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-gitfree-parallel-design.md @@ -0,0 +1,99 @@ +# Git-free `do --parallel` and `benchmark` — design + +Date: 2026-09-26 · Scope: `internal/worktree`, `internal/engine` (headless path), `cmd/orchestra/do.go`, +`cmd/orchestra/benchmark.go`. + +## Intent + +Single runs already work in plain folders (`internal/fsdiff`: snapshot → diff → restore). Parallel `do` +and `benchmark` still refuse to run outside a git repository because they isolate each task in a git +worktree. This change gives them a git-free isolation backend so both work anywhere, while keeping the +behaviour inside a repository byte-for-byte the same. + +## Interface + +`internal/worktree` gains an `Isolator` interface. The existing git `*Manager` satisfies it unchanged; a +new `*CopyIsolator` implements it for plain folders. `New(dir)` picks one automatically. + +```go +type Isolator interface { + Add(id, fromRef string) (Tree, error) // fromRef is ignored by the copy isolator + Diff(t Tree) (string, error) // git-style unified diff of the task's own changes + DiffStat(t Tree) (files, added, removed int, err error) + Merge(t Tree, message string) (conflict bool, err error) + Remove(t Tree) error + Cleanup() +} + +func New(dir string) (Isolator, error) // git repo → *Manager, otherwise → *CopyIsolator +func NewCopy(dir string) (*CopyIsolator, error) +``` + +`Tree` is unchanged (`ID`, `Dir`, `Branch`); the copy isolator leaves `Branch` empty. + +## Copy isolator semantics + +- **Root.** `os.MkdirTemp("", "orchestra-cp-")`, one sub-directory per task. `NewCopy` refuses a temp + root that is itself inside a git work tree (otherwise the headless engine would see a "repo" and + commit into it). +- **Add (copy).** Walk the source directory at `Add` time — so a later dependency wave starts from the + source *after* earlier merges, like `git worktree add HEAD`. Directories matched by + `fsdiff.Excluded(rel, true)` (VCS internals, `node_modules`, `vendor`, build outputs, …) are skipped — + the same rule fsdiff uses for snapshots; no new ignore list. Every other entry is copied: regular files + with their permission bits, symlinks re-created as symlinks (target copied verbatim, never followed), + directories with their permission bits (plus owner rwx so the copy stays removable). Special files are + skipped. + While copying, a per-file *signature* (mode + SHA-256 of content, or `symlink → target`) is recorded: + this is the task's merge baseline. Right after copying, `fsdiff.Capture(copy)` is taken as the task's + diff baseline. Both describe exactly what the agent started from. +- **Diff.** `fsdiff.Diff(copy, baseline)` — the same git-style unified output single runs produce, so + review prompts, colouring and the TUI need nothing new. `DiffStat` counts `diff --git` sections and + `+`/`-` body lines of that diff. +- **Merge (accept).** The touched set is every path fsdiff tracks (regular, non-excluded files) whose + signature in the copy differs from the baseline: added, modified (content or mode) or deleted. Only + paths the reviewer could see in the diff are merged; symlinks and excluded files never are. + 1. *Guard* every touched path: relative, clean, no `..`, not absolute, and no existing parent component + in the source may be a symlink (a write can never escape the source directory). + 2. *Conflict check*: for each touched path the source's **current** signature must equal the baseline + signature (a path absent at baseline must still be absent). Any mismatch — typically an earlier + accepted task in the same wave touching the same file — reports `conflict=true` and **nothing is + written for this task** (all-or-nothing per task). Unlike git, there is no line-level three-way + merge: two tasks editing different hunks of one file conflict. + 3. *Apply*: added/modified files are written via temp file + rename with the copy's mode; deleted files + are removed and directories left empty by a deletion are pruned. +- **Remove / Cleanup.** `Remove` deletes the task's copy; `Cleanup` removes the whole root. Callers + `defer Cleanup()`, so ctrl+c (context cancellation) and error paths clean up exactly as worktrees do. + +## Engine + +`ExecuteHeadless` commits inside a worktree. In a directory that is not a git repository it now snapshots +the directory before the loop (as `Execute` already does) and computes the diff with fsdiff; there is no +commit — the changes stay in the isolated copy until `Merge`. Inside a repository nothing changes. + +## Commands + +- `do --parallel`: the "needs a git repository" error is removed. In a repo the clean-tree check and the + stray-write guard (`git restore` of the base) are unchanged. In a plain folder the stray-write guard + snapshots the base before each wave and restores it with `fsdiff.Restore` if an agent wrote outside + its copy. +- `benchmark` (and `--compare`): the git requirement is removed; each agent gets its own copy; the + leaderboard uses `DiffStat`; "merge the winner" uses `Merge`. Memory recording is unchanged. +- Still git-only: nothing else in these commands. + +## Supervised loop + +Unchanged in shape: agents work in isolation, every result is shown as a diff, and only an explicit +accept merges it into the source directory. The one semantic difference in plain folders is the stricter +per-file conflict rule above. + +## Testing + +All offline, fake agents (shell scripts / in-process), temp dirs that are not repositories. + +- `internal/worktree`: copy fidelity (modes, nested dirs, symlink kept as symlink, excluded dir skipped); + diff for add/modify/delete; merge success; conflict → nothing written; traversal guard (symlinked + parent in the source); `Remove`/`Cleanup` delete the copies; `New` picks the right backend. +- `internal/engine`: `ExecuteHeadless` in a plain directory reports the diff and makes no commit. +- `cmd/orchestra`: `runParallel` in a plain temp dir with two shell-script agents writing different files + → both merged; both writing the same file → the second is reported as a conflict and the source keeps + the first; benchmark flow in a plain folder with two fake agents ranks and merges the winner. From 96e9eb82188a30e77ff6f5defa9202f4f96a0f52 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 19:56:38 +0000 Subject: [PATCH 05/25] router: direct API classifier Add APIClassifier, which classifies a message by calling an llm.Provider directly (system prompt = the shared output contract, temperature 0, 256 max tokens, own 30s timeout) instead of spawning the router agent's CLI. Configured with an optional router.classifier block; BuildRouter uses it only when a model is set and its key env var is non-empty, and otherwise falls back to the CLI classifier without failing startup. router.agent still answers plain questions. Includes the Router v2 design spec. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 +- .../specs/2026-09-26-router-v2-design.md | 109 ++++++++++++++++++ internal/config/config.go | 69 ++++++++++- internal/config/router_test.go | 84 ++++++++++++++ internal/router/classifier_api.go | 52 +++++++++ internal/router/classifier_api_test.go | 80 +++++++++++++ internal/router/classifier_cli.go | 9 +- internal/router/router.go | 9 +- 8 files changed, 410 insertions(+), 12 deletions(-) create mode 100644 docs/superpowers/specs/2026-09-26-router-v2-design.md create mode 100644 internal/config/router_test.go create mode 100644 internal/router/classifier_api.go create mode 100644 internal/router/classifier_api_test.go diff --git a/README.md b/README.md index 093a9ab..7883d52 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,14 @@ router: plan: claude implement: opencode review: claude + # Optional: classify with a cheap hosted model over HTTP instead of spawning the + # router agent's CLI. Used only when the key env var is set; otherwise Orchestra + # quietly falls back to `agent` above (which still answers plain questions). + # classifier: + # provider: anthropic # openai (default) | anthropic + # model: claude-haiku-4-5 + # api_base: "" # optional endpoint override + # api_key_env: "" # optional; defaults to ANTHROPIC_API_KEY / OPENAI_API_KEY agents: - name: claude @@ -277,7 +285,7 @@ internal/llm HTTP LLM providers (openai-compatible, anthropic) behind on internal/patch extract changes from model replies and apply them safely internal/ui terminal styling: gradient banner, spinners, colored diffs (TTY-aware) internal/tui Bubble Tea dashboard: agents / history / benchmarks (read-only monitor) -internal/router AI routing: Classifier (CLI now, API later) → Decision, 3-tier fallback +internal/router AI routing: Classifier (agent CLI or direct API) → Decision, tiered fallback internal/planner decompose a request into ordered steps (+ depends_on for parallel) internal/scheduler bounded-concurrency runner + DAG waves (cycle/blocked detection) internal/worktree git-worktree isolation: branch per task, merge + conflict detection diff --git a/docs/superpowers/specs/2026-09-26-router-v2-design.md b/docs/superpowers/specs/2026-09-26-router-v2-design.md new file mode 100644 index 0000000..dbf9104 --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-router-v2-design.md @@ -0,0 +1,109 @@ +# Router v2 — direct API classifier + history-aware agent resolution + +Date: 2026-09-26 + +## Intent + +1. **Cheaper, faster classification.** Today every message is classified by spawning the router + agent's CLI in query mode (`claude -p …`), which costs seconds and a full agent session. A small + hosted model (e.g. Haiku) called directly over HTTP classifies in well under a second. +2. **Use what Orchestra already knows.** The memory store records which agent's work was accepted in + each directory, and which agent won benchmarks there. When the AI has no strong opinion, that track + record is a better tie-breaker than a static route table. + +Neither change touches the supervised loop: routing only chooses *which* agent runs. Dispatch, +validation, retry and diff review are unchanged, and an `@agent` override still bypasses the router. + +## Part A — APIClassifier + +`internal/router/classifier_api.go` adds `APIClassifier`, implementing `Classifier` by calling an +`llm.Provider` directly: + +- System prompt = the same output contract as the CLI classifier (JSON `intent/agent/reason`, the + four intent definitions, the allowed agent names). User message = the raw message. +- `MaxTokens: 256`, `Temperature: 0`, its own timeout (default 30s, independent of the task timeout). +- Output parsed by the existing lenient `parseClassification` (tolerates fences and prose). +- The router package imports `internal/llm` but never `internal/config`. + +### Config + +```yaml +router: + agent: claude # still the answerer for plain questions + classifier: # optional — omit to keep the CLI classifier + provider: anthropic # openai (default) | anthropic + model: claude-haiku-4-5 + api_base: "" # optional endpoint override + api_key_env: "" # optional; provider default (ANTHROPIC_API_KEY / OPENAI_API_KEY) +``` + +Merged field-by-field over the defaults like the other router fields (defaults leave it empty). + +`BuildRouter` uses the APIClassifier only when `classifier.model` is set **and** the key env var is +non-empty **and** the provider can be constructed. Otherwise it silently falls back to the CLI +classifier — a missing key never fails startup. The answerer stays `router.agent`, so a cheap +classifier can pair with a strong answerer. + +## Part B — history-aware resolution + +Resolution order for a task (plan / implement / review): + +1. the AI's explicit agent suggestion, if healthy; +2. the best agent by track record in this directory, if healthy and with enough samples; +3. the static route for the intent; +4. the default agent; +5. the first healthy registered agent. + +The same order applies when classification fails (degraded to implement with no suggestion). + +The router does not import memory. It defines + +```go +type History interface { + BestAgent(dir string, candidates []string) (name, reason string, ok bool) +} +``` + +set with `(*Router).WithHistory(h)`. A nil History keeps the previous behaviour. The router passes +only the currently healthy agent names as candidates. + +### Scoring (`(*memory.Store).AgentScores` / `BestAgent`) + +For each candidate agent, over rows whose `dir` equals the absolute project directory: + +- **successes** = accepted runs + benchmark wins +- **samples** = all runs + benchmark wins + validation-failed benchmarks + (`valid = 0 AND skipped = 0`). A valid-but-lost benchmark is neutral and not counted. +- **score** = (successes + 1) / (samples + 2) — Laplace-smoothed acceptance rate. + +Pick rules: + +- only candidates passed in are considered; others are ignored; +- a candidate needs **samples ≥ 3**; +- its score must beat 0.5 (more successes than failures) — history never *pushes* a poor agent; +- the highest score wins; if the top score is shared, there is **no pick** (fall through). + +The reason explains the pick, e.g. `claude: 8/10 accepted in this dir` (successes/samples). The +router appends it to the classifier's reason: ` · history: claude: 8/10 …`. + +## Wiring + +- Shell (`orchestra` root) and dashboard: `BuildRouter(reg)`, then `WithHistory(mem)` when the + memory store opened. If it didn't, history is skipped silently. +- `orchestra do` does not use the router (the planner assigns agents per step), so it is unchanged. + +## Error handling + +- API classifier errors (network, auth, timeout, unparsable output) surface as a classification + error, which the router already degrades to "implement on the resolved agent". +- History query errors are swallowed (`ok = false`) — history is advisory. + +## Testing + +- `APIClassifier` with a fake `llm.Provider`: valid JSON, fenced JSON, garbage, provider error; + asserts the request (system prompt contains choices, temperature 0, max tokens). +- Config: classifier merge; BuildRouter falls back to CLI when the key env is empty; uses API when set. +- Router: table test of the resolution order with a fake History and fake agents (`true` / missing + binaries for health). +- Memory: temp-dir sqlite — min-sample threshold, ties, candidate filtering, benchmark weighting, + dir scoping. diff --git a/internal/config/config.go b/internal/config/config.go index 2bcc18e..020e131 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "time" "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/llm" "github.com/crossben/orchestra-code/internal/router" "github.com/crossben/orchestra-code/internal/validate" "gopkg.in/yaml.v3" @@ -53,6 +54,19 @@ type RouterConfig struct { Enabled *bool `yaml:"enabled"` // pointer so "false" differs from unset Agent string `yaml:"agent"` // agent that does classification/answers Routes map[string]string `yaml:"routes"` // intent → agent name (static fallback) + + // Classifier optionally classifies via a hosted LLM directly (no CLI spawn). + // Used only when Model is set and the key env var is non-empty; otherwise + // the router agent classifies. The router agent still answers questions. + Classifier ClassifierConfig `yaml:"classifier"` +} + +// ClassifierConfig configures the direct-API router classifier. +type ClassifierConfig struct { + Provider string `yaml:"provider"` // openai (default) | anthropic + Model string `yaml:"model"` // required to enable the API classifier + APIBase string `yaml:"api_base"` // optional endpoint override + APIKeyEnv string `yaml:"api_key_env"` // env var holding the key (provider default) } // Config is the top-level Orchestra configuration. @@ -300,6 +314,18 @@ func merge(base, user *Config) { } base.Router.Routes[intent] = ag } + if uc := user.Router.Classifier; uc.Provider != "" { + base.Router.Classifier.Provider = uc.Provider + } + if uc := user.Router.Classifier; uc.Model != "" { + base.Router.Classifier.Model = uc.Model + } + if uc := user.Router.Classifier; uc.APIBase != "" { + base.Router.Classifier.APIBase = uc.APIBase + } + if uc := user.Router.Classifier; uc.APIKeyEnv != "" { + base.Router.Classifier.APIKeyEnv = uc.APIKeyEnv + } if user.Timeout != "" { base.Timeout = user.Timeout } @@ -319,8 +345,8 @@ func merge(base, user *Config) { } // BuildRouter constructs the AI router from config against a registry. The -// router (CLI classifier) uses the configured router agent for classification -// and direct answers. +// configured router agent answers plain questions; classification uses the +// direct-API classifier when router.classifier is usable, else the router agent. func (c *Config) BuildRouter(reg *agent.Registry) (*router.Router, error) { ra, ok := reg.Get(c.RouterAgent()) if !ok { @@ -330,13 +356,46 @@ func (c *Config) BuildRouter(reg *agent.Registry) (*router.Router, error) { if !ok { return nil, fmt.Errorf("router agent %q cannot answer questions (no query support)", c.RouterAgent()) } - cls, err := router.NewCLIClassifier(ra, reg.Names(), c.TimeoutDuration()) - if err != nil { - return nil, err + var cls router.Classifier + if api := c.apiClassifier(reg.Names()); api != nil { + cls = api + } else { + cli, err := router.NewCLIClassifier(ra, reg.Names(), c.TimeoutDuration()) + if err != nil { + return nil, err + } + cls = cli } return router.New(cls, answerer, reg, c.Router.Routes, c.DefaultAgent), nil } +// apiClassifier returns the direct-API classifier, or nil when it is not +// configured (no model), has no key in the environment, or names an unusable +// provider. Nil means "fall back to the CLI classifier" — never a startup error. +func (c *Config) apiClassifier(choices []string) *router.APIClassifier { + cc := c.Router.Classifier + if strings.TrimSpace(cc.Model) == "" { + return nil + } + provider := cc.Provider + if provider == "" { + provider = "openai" + } + keyEnv := cc.APIKeyEnv + if keyEnv == "" { + keyEnv = llm.DefaultKeyEnv(provider) + } + key := os.Getenv(keyEnv) + if key == "" { + return nil + } + prov, err := llm.New(provider, cc.APIBase, key, cc.Model, nil) + if err != nil { + return nil + } + return router.NewAPIClassifier(prov, choices, router.DefaultClassifierTimeout) +} + // BuildRegistry turns the config's agents into a live agent.Registry. Agents // with type "api" become APIAgent instances (HTTP-backed); everything else is // a CLIAgent. A misconfigured agent is skipped with a warning rather than diff --git a/internal/config/router_test.go b/internal/config/router_test.go new file mode 100644 index 0000000..24dde61 --- /dev/null +++ b/internal/config/router_test.go @@ -0,0 +1,84 @@ +package config + +import ( + "testing" + + "github.com/crossben/orchestra-code/internal/router" +) + +const classifierYAML = ` +router: + classifier: + provider: anthropic + model: claude-haiku-4-5 + api_key_env: ORCH_TEST_CLASSIFIER_KEY +` + +func TestLoadMergesRouterClassifier(t *testing.T) { + cfg, err := parseForTest(t, classifierYAML) + if err != nil { + t.Fatal(err) + } + got := cfg.Router.Classifier + if got.Provider != "anthropic" || got.Model != "claude-haiku-4-5" || got.APIKeyEnv != "ORCH_TEST_CLASSIFIER_KEY" { + t.Fatalf("classifier not merged: %+v", got) + } + // Other router defaults survive the partial override. + if cfg.Router.Agent != "claude" || cfg.Router.Routes["implement"] != "opencode" { + t.Fatalf("router defaults lost: %+v", cfg.Router) + } +} + +func TestBuildClassifierFallsBackWithoutKey(t *testing.T) { + t.Setenv("ORCH_TEST_CLASSIFIER_KEY", "") + cfg, err := parseForTest(t, classifierYAML) + if err != nil { + t.Fatal(err) + } + r, err := cfg.BuildRouter(cfg.BuildRegistry()) + if err != nil { + t.Fatalf("missing key must not fail startup: %v", err) + } + if _, ok := r.Classifier().(*router.CLIClassifier); !ok { + t.Fatalf("want CLI classifier fallback, got %T", r.Classifier()) + } +} + +func TestBuildClassifierUsesAPIWhenKeySet(t *testing.T) { + t.Setenv("ORCH_TEST_CLASSIFIER_KEY", "sk-test") + cfg, err := parseForTest(t, classifierYAML) + if err != nil { + t.Fatal(err) + } + r, err := cfg.BuildRouter(cfg.BuildRegistry()) + if err != nil { + t.Fatal(err) + } + if _, ok := r.Classifier().(*router.APIClassifier); !ok { + t.Fatalf("want API classifier, got %T", r.Classifier()) + } +} + +func TestBuildClassifierFallsBackOnBadProvider(t *testing.T) { + t.Setenv("ORCH_TEST_CLASSIFIER_KEY", "sk-test") + cfg := Default() + cfg.Router.Classifier = ClassifierConfig{Provider: "nosuch", Model: "m", APIKeyEnv: "ORCH_TEST_CLASSIFIER_KEY"} + r, err := cfg.BuildRouter(cfg.BuildRegistry()) + if err != nil { + t.Fatal(err) + } + if _, ok := r.Classifier().(*router.CLIClassifier); !ok { + t.Fatalf("want CLI classifier fallback, got %T", r.Classifier()) + } +} + +func TestBuildClassifierDefaultIsCLI(t *testing.T) { + cfg := Default() + r, err := cfg.BuildRouter(cfg.BuildRegistry()) + if err != nil { + t.Fatal(err) + } + if _, ok := r.Classifier().(*router.CLIClassifier); !ok { + t.Fatalf("default must stay the CLI classifier, got %T", r.Classifier()) + } +} diff --git a/internal/router/classifier_api.go b/internal/router/classifier_api.go new file mode 100644 index 0000000..d9a8fdd --- /dev/null +++ b/internal/router/classifier_api.go @@ -0,0 +1,52 @@ +package router + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/crossben/orchestra-code/internal/llm" +) + +// DefaultClassifierTimeout bounds one API classification call. It is separate +// from the task timeout: classifying should take seconds, not minutes. +const DefaultClassifierTimeout = 30 * time.Second + +// apiClassifierMaxTokens caps the reply — the verdict is a tiny JSON object. +const apiClassifierMaxTokens = 256 + +// APIClassifier classifies messages by calling a hosted LLM directly over HTTP +// (no agent CLI is spawned). A small, cheap model is enough; the answerer for +// plain questions stays the router agent. +type APIClassifier struct { + prov llm.Provider + choices []string + timeout time.Duration +} + +// NewAPIClassifier wraps a provider. choices is the list of agent names the +// classifier may suggest; timeout <= 0 uses DefaultClassifierTimeout. +func NewAPIClassifier(p llm.Provider, choices []string, timeout time.Duration) *APIClassifier { + if timeout <= 0 { + timeout = DefaultClassifierTimeout + } + return &APIClassifier{prov: p, choices: choices, timeout: timeout} +} + +// Classify sends the output contract as the system prompt and the message as +// the user turn, then leniently parses the JSON verdict. +func (c *APIClassifier) Classify(ctx context.Context, message, _ string) (Classification, error) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + resp, err := c.prov.Complete(ctx, llm.Request{ + System: fmt.Sprintf(classifyContract, strings.Join(c.choices, ", ")), + Messages: []llm.Message{{Role: "user", Content: message}}, + MaxTokens: apiClassifierMaxTokens, + Temperature: 0, + }) + if err != nil { + return Classification{}, fmt.Errorf("%s classifier: %w", c.prov.Name(), err) + } + return parseClassification(resp.Text) +} diff --git a/internal/router/classifier_api_test.go b/internal/router/classifier_api_test.go new file mode 100644 index 0000000..0e39b24 --- /dev/null +++ b/internal/router/classifier_api_test.go @@ -0,0 +1,80 @@ +package router + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/crossben/orchestra-code/internal/llm" +) + +// fakeProvider is an offline llm.Provider returning a canned reply. +type fakeProvider struct { + text string + err error + got llm.Request +} + +func (f *fakeProvider) Name() string { return "fake" } + +func (f *fakeProvider) Complete(ctx context.Context, req llm.Request) (llm.Response, error) { + f.got = req + if _, ok := ctx.Deadline(); !ok { + return llm.Response{}, errors.New("classifier must set a deadline") + } + return llm.Response{Text: f.text}, f.err +} + +func TestAPIClassifier(t *testing.T) { + cases := []struct { + name string + text string + err error + want Classification + wantErr bool + }{ + {name: "valid json", text: `{"intent":"implement","agent":"opencode","reason":"code change"}`, + want: Classification{Intent: IntentImplement, Agent: "opencode", Reason: "code change"}}, + {name: "fenced json", text: "```json\n{\"intent\": \"Question\", \"agent\": \" \", \"reason\": \"asks\"}\n```", + want: Classification{Intent: IntentQuestion, Reason: "asks"}}, + {name: "garbage", text: "I think you want to implement it", wantErr: true}, + {name: "provider error", err: errors.New("401 unauthorized"), wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := &fakeProvider{text: tc.text, err: tc.err} + c := NewAPIClassifier(p, []string{"claude", "opencode"}, time.Second) + got, err := c.Classify(context.Background(), "add pagination", "/tmp") + if tc.wantErr { + if err == nil { + t.Fatalf("want error, got %+v", got) + } + return + } + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + if p.got.Temperature != 0 || p.got.MaxTokens != apiClassifierMaxTokens { + t.Fatalf("unexpected request params: %+v", p.got) + } + if !strings.Contains(p.got.System, "claude, opencode") { + t.Fatalf("system prompt lacks agent choices: %q", p.got.System) + } + if len(p.got.Messages) != 1 || p.got.Messages[0].Role != "user" || p.got.Messages[0].Content != "add pagination" { + t.Fatalf("unexpected messages: %+v", p.got.Messages) + } + }) + } +} + +func TestAPIClassifierDefaultTimeout(t *testing.T) { + c := NewAPIClassifier(&fakeProvider{}, nil, 0) + if c.timeout != DefaultClassifierTimeout { + t.Fatalf("timeout = %v, want %v", c.timeout, DefaultClassifierTimeout) + } +} diff --git a/internal/router/classifier_cli.go b/internal/router/classifier_cli.go index 5d36087..3e104ac 100644 --- a/internal/router/classifier_cli.go +++ b/internal/router/classifier_cli.go @@ -30,7 +30,9 @@ func NewCLIClassifier(a agent.Agent, choices []string, timeout time.Duration) (* return &CLIClassifier{agent: q, name: a.Name(), choices: choices, timeout: timeout}, nil } -const classifyTemplate = `You are the router for a coding assistant. Classify the user's message and respond with ONLY a JSON object — no prose, no markdown fences: +// classifyContract is the output contract shared by every classifier. %s is the +// comma-separated list of agent names the classifier may suggest. +const classifyContract = `You are the router for a coding assistant. Classify the user's message and respond with ONLY a JSON object — no prose, no markdown fences: {"intent": "question|plan|implement|review", "agent": "", "reason": ""} Definitions: @@ -38,9 +40,10 @@ Definitions: - "plan": the user wants a large task broken into steps. - "implement": the user wants code written or modified. - "review": the user wants existing code reviewed or critiqued. -Set "agent" only if one is clearly most suitable; otherwise leave it empty. +Set "agent" only if one is clearly most suitable; otherwise leave it empty.` -User message: %s` +// classifyTemplate is the single-prompt form used by the CLI classifier. +const classifyTemplate = classifyContract + "\n\nUser message: %s" // Classify queries the agent and parses its JSON verdict. It uses a quiet query // (stdout only, stderr discarded) so classification never prints stray output — diff --git a/internal/router/router.go b/internal/router/router.go index 20c7cb8..ac0e981 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -2,9 +2,9 @@ // decides what to do with it — answer a plain question directly, or dispatch a // coding task to the best available agent. // -// Classification is pluggable via the Classifier interface. M4 ships a CLI-based -// classifier (an agent in query mode); a direct-API classifier can drop in later -// without touching the resolution logic here. +// Classification is pluggable via the Classifier interface: CLIClassifier asks +// an agent in query mode; APIClassifier calls a hosted LLM directly over HTTP. +// Neither affects the resolution logic here. // // Resolution is three-tier and never blocks: AI suggestion → static routes → // default agent. Only healthy (installed) agents are chosen. @@ -64,6 +64,9 @@ func New(cls Classifier, answerer agent.Querier, reg *agent.Registry, routes map return &Router{cls: cls, answerer: answerer, reg: reg, routes: routes, fallback: fallback} } +// Classifier returns the classifier in use (useful for diagnostics and tests). +func (r *Router) Classifier() Classifier { return r.cls } + // Route classifies the message and resolves it to a Decision. Classification // failure is not fatal — it degrades to an implement task on the default agent. func (r *Router) Route(ctx context.Context, message, dir string) Decision { From 1a459b57f76a60fe3988aa6f90ea0beaaabed70c Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 19:58:19 +0000 Subject: [PATCH 06/25] worktree: add Isolator interface and a copy isolator for plain folders The git worktree Manager now satisfies an Isolator interface; a new CopyIsolator gives each task a directory copy (fsdiff exclusion rules, modes and symlinks preserved), diffs it with fsdiff, and merges accepted changes back all-or-nothing with per-file conflict detection and traversal-guarded writes. worktree.New picks the backend. engine.ExecuteHeadless diffs against a snapshot and skips the commit when its directory is not a git repository; inside a repo it is unchanged. Co-Authored-By: Claude Opus 5.5 --- internal/engine/engine.go | 15 +- internal/engine/engine_test.go | 23 ++ internal/worktree/copy.go | 462 +++++++++++++++++++++++++++++++++ internal/worktree/copy_test.go | 323 +++++++++++++++++++++++ internal/worktree/isolator.go | 42 +++ 5 files changed, 864 insertions(+), 1 deletion(-) create mode 100644 internal/worktree/copy.go create mode 100644 internal/worktree/copy_test.go create mode 100644 internal/worktree/isolator.go diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 21d4476..9d47b71 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -391,6 +391,10 @@ func runLoop(ctx context.Context, opts Options) (out Outcome, err error) { // ExecuteHeadless runs the loop non-interactively and commits any resulting // changes to the current branch (used inside a per-task worktree during parallel // execution). No accept/reject prompt — review happens later at merge time. +// +// Outside a git repository (a per-task directory copy) there is nothing to +// commit: the changes are diffed against a pre-run snapshot and left in place +// for the isolator to merge on accept. func ExecuteHeadless(ctx context.Context, opts Options) (out Outcome, err error) { defer func() { if err == nil { @@ -398,11 +402,16 @@ func ExecuteHeadless(ctx context.Context, opts Options) (out Outcome, err error) } }() + before, inRepo, err := captureBaseline(opts.Dir) + if err != nil { + return out, err + } + out, err = runLoop(ctx, opts) if err != nil { return out, err } - diff, derr := gitutil.Diff(opts.Dir) + diff, derr := workingTreeDiff(opts.Dir, before) if derr != nil { return out, fmt.Errorf("compute diff: %w", derr) } @@ -412,6 +421,10 @@ func ExecuteHeadless(ctx context.Context, opts Options) (out Outcome, err error) } out.HadChanges = true out.Diff = diff + if !inRepo { + opts.logf("%s changes kept in the isolated copy", ui.Success("✓")) + return out, nil + } if err := gitutil.Commit(opts.Dir, commitMessage(opts.Prompt)); err != nil { return out, fmt.Errorf("commit changes: %w", err) } diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index b07dcab..e517ba6 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -246,3 +246,26 @@ echo "attempt $n" t.Fatalf("output not streamed: %q", s) } } + +func TestExecuteHeadlessNonGitDiffsWithoutCommit(t *testing.T) { + dir := t.TempDir() // deliberately NOT a git repository + mustWrite(t, filepath.Join(dir, "pre.txt"), "before\n") + + out, err := ExecuteHeadless(context.Background(), optsFor(&writeAgent{name: "fake", path: "created.txt", content: "hi\n"}, dir)) + if err != nil { + t.Fatalf("ExecuteHeadless: %v", err) + } + if !out.HadChanges { + t.Fatal("expected changes") + } + if !strings.Contains(out.Diff, "diff --git a/created.txt b/created.txt") || !strings.Contains(out.Diff, "+hi") { + t.Fatalf("diff missing the new file:\n%s", out.Diff) + } + if strings.Contains(out.Diff, "pre.txt") { + t.Fatalf("diff should only show the run's changes:\n%s", out.Diff) + } + if _, err := os.Stat(filepath.Join(dir, ".git")); !os.IsNotExist(err) { + t.Fatal("headless run in a plain dir must not create a repository") + } + mustContain(t, filepath.Join(dir, "created.txt"), "hi") +} diff --git a/internal/worktree/copy.go b/internal/worktree/copy.go new file mode 100644 index 0000000..1b687cb --- /dev/null +++ b/internal/worktree/copy.go @@ -0,0 +1,462 @@ +package worktree + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/crossben/orchestra-code/internal/fsdiff" + "github.com/crossben/orchestra-code/internal/gitutil" +) + +// CopyIsolator isolates tasks in plain directory copies, for working +// directories that are not git repositories. Each task gets a full copy of the +// source (minus the directories fsdiff never tracks); its changes are diffed +// with fsdiff against the state it started from, and merged back file by file +// only if none of the touched files changed in the source meanwhile. +type CopyIsolator struct { + src string // absolute source directory + root string // temp dir holding the copies + + mu sync.Mutex + trees map[string]*copyTree // by tree ID +} + +// copyTree is the bookkeeping for one copy. +type copyTree struct { + dir string + baseline *fsdiff.Snapshot // fsdiff capture of the fresh copy (diff baseline) + sigs map[string]string +} + +// NewCopy prepares a copy root for src. +func NewCopy(src string) (*CopyIsolator, error) { + abs, err := filepath.Abs(src) + if err != nil { + return nil, err + } + root, err := os.MkdirTemp("", "orchestra-cp-") + if err != nil { + return nil, err + } + // The headless engine treats a git work tree as "commit here"; a copy + // that happened to land inside one would commit into a stranger's repo. + if gitutil.IsRepo(root) { + _ = os.RemoveAll(root) + return nil, fmt.Errorf("temp directory %s is inside a git repository; set TMPDIR elsewhere", filepath.Dir(root)) + } + return &CopyIsolator{src: abs, root: root, trees: map[string]*copyTree{}}, nil +} + +// Add copies the source's current state into a fresh directory for task id. +// fromRef is ignored (there are no refs outside git). +func (c *CopyIsolator) Add(id, _ string) (Tree, error) { + if !safeRel(id) || strings.Contains(id, "/") { + return Tree{}, fmt.Errorf("invalid task id %q", id) + } + dir := filepath.Join(c.root, id) + if err := os.Mkdir(dir, 0o700); err != nil { + return Tree{}, fmt.Errorf("create copy %s: %w", id, err) + } + sigs, err := copyTreeFiles(c.src, dir, c.root) + if err != nil { + _ = os.RemoveAll(dir) + return Tree{}, fmt.Errorf("copy %s: %w", id, err) + } + base, err := fsdiff.Capture(dir) + if err != nil { + _ = os.RemoveAll(dir) + return Tree{}, fmt.Errorf("snapshot copy %s: %w", id, err) + } + c.mu.Lock() + c.trees[id] = ©Tree{dir: dir, baseline: base, sigs: sigs} + c.mu.Unlock() + return Tree{ID: id, Dir: dir}, nil +} + +func (c *CopyIsolator) tree(t Tree) (*copyTree, error) { + c.mu.Lock() + defer c.mu.Unlock() + ct, ok := c.trees[t.ID] + if !ok || ct.dir != t.Dir { + return nil, fmt.Errorf("unknown copy %q", t.ID) + } + return ct, nil +} + +// Diff returns the copy's changes since it was created, in git's unified format. +func (c *CopyIsolator) Diff(t Tree) (string, error) { + ct, err := c.tree(t) + if err != nil { + return "", err + } + return fsdiff.Diff(ct.dir, ct.baseline) +} + +// DiffStat counts files and +/- lines in Diff's output. +func (c *CopyIsolator) DiffStat(t Tree) (files, added, removed int, err error) { + d, err := c.Diff(t) + if err != nil { + return 0, 0, 0, err + } + for _, line := range strings.Split(d, "\n") { + switch { + case strings.HasPrefix(line, "diff --git "): + files++ + case strings.HasPrefix(line, "+++ "), strings.HasPrefix(line, "--- "): + case strings.HasPrefix(line, "+"): + added++ + case strings.HasPrefix(line, "-"): + removed++ + } + } + return files, added, removed, nil +} + +// Merge applies the copy's added/modified/deleted files to the source. Every +// touched path is checked first: it must be a safe relative path that does +// not pass through a symlink in the source (error otherwise), and its current +// state in the source must equal the state the copy started from (conflict +// otherwise). Only when all checks pass is anything written. +func (c *CopyIsolator) Merge(t Tree, _ string) (conflict bool, err error) { + ct, err := c.tree(t) + if err != nil { + return false, err + } + now, err := trackedSigs(ct.dir) + if err != nil { + return false, fmt.Errorf("scan copy %s: %w", t.ID, err) + } + + var touched []string + for p, sig := range now { + if ct.sigs[p] != sig { + touched = append(touched, p) // added or modified + } + } + for p, sig := range ct.sigs { + if _, ok := now[p]; !ok && isFileSig(sig) && fsdiffTracks(p) { + touched = append(touched, p) // deleted + } + } + sort.Strings(touched) + + // 1. Guard every path before looking at or writing anything. + for _, p := range touched { + if err := c.guard(p); err != nil { + return false, err + } + } + // 2. Conflict check: the source must still hold the baseline version. + for _, p := range touched { + cur, err := sigOf(filepath.Join(c.src, filepath.FromSlash(p))) + if err != nil { + return false, err + } + if cur != ct.sigs[p] { + return true, nil + } + } + // 3. Apply. + for _, p := range touched { + target := filepath.Join(c.src, filepath.FromSlash(p)) + if _, ok := now[p]; !ok { + if err := os.Remove(target); err != nil && !os.IsNotExist(err) { + return false, fmt.Errorf("remove %s: %w", p, err) + } + pruneEmpty(c.src, filepath.Dir(target)) + continue + } + if err := copyFileAtomic(filepath.Join(ct.dir, filepath.FromSlash(p)), target); err != nil { + return false, fmt.Errorf("write %s: %w", p, err) + } + } + return false, nil +} + +// guard rejects paths that could write outside the source directory: anything +// not clean and relative, or whose existing parent directories in the source +// include a symlink. +func (c *CopyIsolator) guard(p string) error { + if !safeRel(p) { + return fmt.Errorf("refusing unsafe path %q", p) + } + parts := strings.Split(p, "/") + cur := c.src + for _, part := range parts[:len(parts)-1] { + cur = filepath.Join(cur, part) + info, err := os.Lstat(cur) + if os.IsNotExist(err) { + return nil // the rest will be created as real directories + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("refusing to write %q: %s is not a plain directory in the source", p, cur) + } + } + return nil +} + +// Remove deletes one copy. +func (c *CopyIsolator) Remove(t Tree) error { + c.mu.Lock() + delete(c.trees, t.ID) + c.mu.Unlock() + if t.Dir == "" || !strings.HasPrefix(t.Dir, c.root+string(filepath.Separator)) { + return nil + } + return os.RemoveAll(t.Dir) +} + +// Cleanup deletes every copy and the root. +func (c *CopyIsolator) Cleanup() { + c.mu.Lock() + c.trees = map[string]*copyTree{} + c.mu.Unlock() + _ = os.RemoveAll(c.root) +} + +// safeRel reports whether p is a clean, slash-separated relative path that +// stays inside its root. +func safeRel(p string) bool { + if p == "" || strings.HasPrefix(p, "/") || strings.Contains(p, "\\") { + return false + } + if path.Clean(p) != p || p == "." || p == ".." || strings.HasPrefix(p, "../") { + return false + } + return true +} + +// fsdiffTracks reports whether fsdiff would show p in a diff: no path +// component is an excluded directory and the file itself is not excluded. +func fsdiffTracks(p string) bool { + parts := strings.Split(p, "/") + for i := range parts[:len(parts)-1] { + if fsdiff.Excluded(strings.Join(parts[:i+1], "/"), true) { + return false + } + } + return !fsdiff.Excluded(p, false) +} + +// Signatures: "f:" for regular files, "l:" for symlinks, +// "" for absent paths. +func isFileSig(s string) bool { return strings.HasPrefix(s, "f") } + +func sigOf(p string) (string, error) { + info, err := os.Lstat(p) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + switch { + case info.Mode()&os.ModeSymlink != 0: + target, err := os.Readlink(p) + if err != nil { + return "", err + } + return "l:" + target, nil + case info.Mode().IsRegular(): + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("f%o:%s", info.Mode().Perm(), hex.EncodeToString(h.Sum(nil))), nil + default: + return "d", nil // a directory or special file where a file is expected + } +} + +// trackedSigs signs every regular file in dir that fsdiff tracks (the files a +// reviewer can see in the diff) — the only files Merge will ever touch. +func trackedSigs(dir string) (map[string]string, error) { + sigs := map[string]string{} + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if rel == "." { + return nil + } + if d.IsDir() { + if fsdiff.Excluded(rel, true) { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() || fsdiff.Excluded(rel, false) { + return nil + } + s, err := sigOf(p) + if err != nil { + return err + } + sigs[rel] = s + return nil + }) + return sigs, err +} + +// copyTreeFiles copies src into dst (which exists), skipping directories fsdiff +// excludes (and skip, the copy root, should it live inside src), and returns +// the signature of every copied file and symlink. +func copyTreeFiles(src, dst, skip string) (map[string]string, error) { + sigs := map[string]string{} + var dirModes []struct { + path string + mode os.FileMode + } + err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + if rel == "." { + return nil + } + slash := filepath.ToSlash(rel) + target := filepath.Join(dst, rel) + info, err := d.Info() + if err != nil { + return err + } + switch { + case d.IsDir(): + if p == skip || fsdiff.Excluded(slash, true) { + return filepath.SkipDir + } + if err := os.Mkdir(target, 0o700); err != nil { + return err + } + dirModes = append(dirModes, struct { + path string + mode os.FileMode + }{target, info.Mode().Perm() | 0o700}) + case info.Mode()&os.ModeSymlink != 0: + link, err := os.Readlink(p) + if err != nil { + return err + } + if err := os.Symlink(link, target); err != nil { + return err + } + sigs[slash] = "l:" + link + case info.Mode().IsRegular(): + s, err := copyFileSig(p, target, info.Mode().Perm()) + if err != nil { + return err + } + sigs[slash] = s + } + return nil // special files are skipped + }) + if err != nil { + return nil, err + } + for i := len(dirModes) - 1; i >= 0; i-- { + _ = os.Chmod(dirModes[i].path, dirModes[i].mode) + } + return sigs, nil +} + +// copyFileSig copies one regular file (content + mode), returning the +// signature of exactly the bytes copied. +func copyFileSig(src, dst string, mode os.FileMode) (string, error) { + in, err := os.Open(src) + if err != nil { + return "", err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return "", err + } + h := sha256.New() + if _, err := io.Copy(io.MultiWriter(out, h), in); err != nil { + out.Close() + return "", err + } + if err := out.Close(); err != nil { + return "", err + } + if err := os.Chmod(dst, mode); err != nil { // defeat umask + return "", err + } + return fmt.Sprintf("f%o:%s", mode, hex.EncodeToString(h.Sum(nil))), nil +} + +// copyFileAtomic writes src's content and mode to dst via a temp file in dst's +// directory and a rename, creating parent directories as needed. +func copyFileAtomic(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + data, err := os.ReadFile(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), ".orchestra-merge-*") + if err != nil { + return err + } + name := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(name) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(name) + return err + } + if err := os.Chmod(name, info.Mode().Perm()); err != nil { + os.Remove(name) + return err + } + if err := os.Rename(name, dst); err != nil { + os.Remove(name) + return err + } + return nil +} + +// pruneEmpty removes dir and its parents while they are empty, stopping at root. +func pruneEmpty(root, dir string) { + for dir != root && strings.HasPrefix(dir, root+string(filepath.Separator)) { + if err := os.Remove(dir); err != nil { + return // not empty (or not removable): stop + } + dir = filepath.Dir(dir) + } +} diff --git a/internal/worktree/copy_test.go b/internal/worktree/copy_test.go new file mode 100644 index 0000000..adfcb3e --- /dev/null +++ b/internal/worktree/copy_test.go @@ -0,0 +1,323 @@ +package worktree + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// plainDir builds a non-git source folder: +// +// keep.txt "keep\n" 0644 +// run.sh "#!/bin/sh\n" 0755 +// sub/deep/n.txt "nested\n" 0600 +// link -> keep.txt (symlink) +// node_modules/x.js (excluded by fsdiff rules) +func plainDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + write(t, dir, "keep.txt", "keep\n", 0o644) + write(t, dir, "run.sh", "#!/bin/sh\n", 0o755) + write(t, dir, "sub/deep/n.txt", "nested\n", 0o600) + write(t, dir, "node_modules/x.js", "heavy\n", 0o644) + if err := os.Symlink("keep.txt", filepath.Join(dir, "link")); err != nil { + t.Fatal(err) + } + return dir +} + +func write(t *testing.T, dir, rel, content string, mode os.FileMode) { + t.Helper() + p := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { // defeat umask + t.Fatal(err) + } +} + +func read(t *testing.T, dir, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + return string(b) +} + +func newCopy(t *testing.T, src string) *CopyIsolator { + t.Helper() + c, err := NewCopy(src) + if err != nil { + t.Fatal(err) + } + t.Cleanup(c.Cleanup) + return c +} + +func TestCopyFidelity(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + tree, err := c.Add("a", "HEAD") + if err != nil { + t.Fatal(err) + } + if tree.Dir == src || !strings.HasPrefix(tree.Dir, c.root) { + t.Fatalf("copy dir %q not under root %q", tree.Dir, c.root) + } + if got := read(t, tree.Dir, "sub/deep/n.txt"); got != "nested\n" { + t.Fatalf("nested content = %q", got) + } + for rel, want := range map[string]os.FileMode{"keep.txt": 0o644, "run.sh": 0o755, "sub/deep/n.txt": 0o600} { + info, err := os.Stat(filepath.Join(tree.Dir, rel)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != want { + t.Errorf("%s mode = %v, want %v", rel, info.Mode().Perm(), want) + } + } + info, err := os.Lstat(filepath.Join(tree.Dir, "link")) + if err != nil { + t.Fatal(err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatal("link was not copied as a symlink") + } + if target, _ := os.Readlink(filepath.Join(tree.Dir, "link")); target != "keep.txt" { + t.Fatalf("symlink target = %q", target) + } + if _, err := os.Stat(filepath.Join(tree.Dir, "node_modules")); !os.IsNotExist(err) { + t.Fatal("excluded dir node_modules should not be copied") + } +} + +func TestCopyDiffAddModifyDelete(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + tree, err := c.Add("a", "HEAD") + if err != nil { + t.Fatal(err) + } + if d, err := c.Diff(tree); err != nil || d != "" { + t.Fatalf("fresh copy diff = %q, %v; want empty", d, err) + } + write(t, tree.Dir, "added.txt", "new\n", 0o644) + write(t, tree.Dir, "keep.txt", "changed\n", 0o644) + if err := os.Remove(filepath.Join(tree.Dir, "sub/deep/n.txt")); err != nil { + t.Fatal(err) + } + d, err := c.Diff(tree) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "diff --git a/added.txt b/added.txt\nnew file mode 100644", + "+new", + "diff --git a/keep.txt b/keep.txt", + "-keep\n+changed", + "diff --git a/sub/deep/n.txt b/sub/deep/n.txt\ndeleted file mode 100644", + "-nested", + } { + if !strings.Contains(d, want) { + t.Errorf("diff missing %q:\n%s", want, d) + } + } + files, added, removed, err := c.DiffStat(tree) + if err != nil { + t.Fatal(err) + } + if files != 3 || added != 2 || removed != 2 { + t.Fatalf("DiffStat = %d files +%d -%d, want 3 +2 -2", files, added, removed) + } + // The source is untouched until merge. + if got := read(t, src, "keep.txt"); got != "keep\n" { + t.Fatalf("source modified before merge: %q", got) + } +} + +func TestCopyMergeApplies(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + tree, _ := c.Add("a", "HEAD") + write(t, tree.Dir, "pkg/added.txt", "new\n", 0o755) + write(t, tree.Dir, "keep.txt", "changed\n", 0o644) + if err := os.Remove(filepath.Join(tree.Dir, "sub/deep/n.txt")); err != nil { + t.Fatal(err) + } + + conflict, err := c.Merge(tree, "merge a") + if err != nil || conflict { + t.Fatalf("merge = conflict %v, err %v", conflict, err) + } + if got := read(t, src, "pkg/added.txt"); got != "new\n" { + t.Fatalf("added = %q", got) + } + if info, _ := os.Stat(filepath.Join(src, "pkg/added.txt")); info.Mode().Perm() != 0o755 { + t.Errorf("added mode = %v", info.Mode().Perm()) + } + if got := read(t, src, "keep.txt"); got != "changed\n" { + t.Fatalf("modified = %q", got) + } + if _, err := os.Stat(filepath.Join(src, "sub")); !os.IsNotExist(err) { + t.Fatal("deleting the only nested file should prune the emptied dirs") + } + // Untouched / excluded content survives. + if got := read(t, src, "node_modules/x.js"); got != "heavy\n" { + t.Fatalf("excluded file changed: %q", got) + } + if target, _ := os.Readlink(filepath.Join(src, "link")); target != "keep.txt" { + t.Fatal("symlink in source disturbed") + } +} + +func TestCopyMergeConflictWritesNothing(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + a, _ := c.Add("a", "HEAD") + b, _ := c.Add("b", "HEAD") + + write(t, a.Dir, "keep.txt", "from a\n", 0o644) + write(t, b.Dir, "keep.txt", "from b\n", 0o644) + write(t, b.Dir, "b-only.txt", "b\n", 0o644) + + if conflict, err := c.Merge(a, "a"); err != nil || conflict { + t.Fatalf("first merge: conflict %v err %v", conflict, err) + } + conflict, err := c.Merge(b, "b") + if err != nil { + t.Fatal(err) + } + if !conflict { + t.Fatal("expected a conflict on keep.txt") + } + if got := read(t, src, "keep.txt"); got != "from a\n" { + t.Fatalf("keep.txt = %q, want a's version", got) + } + if _, err := os.Stat(filepath.Join(src, "b-only.txt")); !os.IsNotExist(err) { + t.Fatal("conflicting task must write nothing (all-or-nothing)") + } +} + +func TestCopyMergeConflictOnAddedPath(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + a, _ := c.Add("a", "HEAD") + b, _ := c.Add("b", "HEAD") + write(t, a.Dir, "same.txt", "a\n", 0o644) + write(t, b.Dir, "same.txt", "b\n", 0o644) + if conflict, _ := c.Merge(a, "a"); conflict { + t.Fatal("unexpected conflict") + } + if conflict, _ := c.Merge(b, "b"); !conflict { + t.Fatal("both adding the same path must conflict") + } + if got := read(t, src, "same.txt"); got != "a\n" { + t.Fatalf("same.txt = %q", got) + } +} + +func TestCopyMergeTraversalGuard(t *testing.T) { + src := t.TempDir() + outside := t.TempDir() + write(t, src, "f.txt", "f\n", 0o644) + c := newCopy(t, src) + tree, _ := c.Add("a", "HEAD") + + // The agent creates a real directory "esc" in its copy; meanwhile the + // source grows a symlink "esc" pointing outside. Writing through it would + // escape the source tree, so merge must refuse and write nothing. + write(t, tree.Dir, "esc/pwn.txt", "pwn\n", 0o644) + write(t, tree.Dir, "ok.txt", "ok\n", 0o644) + if err := os.Symlink(outside, filepath.Join(src, "esc")); err != nil { + t.Fatal(err) + } + conflict, err := c.Merge(tree, "a") + if err == nil && !conflict { + t.Fatal("merge through a symlinked parent must be refused") + } + if _, err := os.Stat(filepath.Join(outside, "pwn.txt")); !os.IsNotExist(err) { + t.Fatal("write escaped the source directory") + } + if _, err := os.Stat(filepath.Join(src, "ok.txt")); !os.IsNotExist(err) { + t.Fatal("a refused merge must write nothing") + } +} + +func TestSafeRel(t *testing.T) { + for _, bad := range []string{"", ".", "..", "../x", "a/../../x", "/etc/passwd", "a/./b"} { + if safeRel(bad) { + t.Errorf("safeRel(%q) = true", bad) + } + } + for _, ok := range []string{"a", "a/b.txt", ".hidden/x"} { + if !safeRel(ok) { + t.Errorf("safeRel(%q) = false", ok) + } + } +} + +func TestCopyLaterAddSeesMergedState(t *testing.T) { + src := plainDir(t) + c := newCopy(t, src) + a, _ := c.Add("a", "HEAD") + write(t, a.Dir, "wave1.txt", "one\n", 0o644) + if conflict, err := c.Merge(a, "a"); err != nil || conflict { + t.Fatal(conflict, err) + } + b, _ := c.Add("b", "HEAD") + if got := read(t, b.Dir, "wave1.txt"); got != "one\n" { + t.Fatalf("second-wave copy missing merged file: %q", got) + } + if d, _ := c.Diff(b); d != "" { + t.Fatalf("second-wave diff should start empty, got:\n%s", d) + } +} + +func TestCopyRemoveAndCleanup(t *testing.T) { + src := plainDir(t) + c, err := NewCopy(src) + if err != nil { + t.Fatal(err) + } + a, _ := c.Add("a", "HEAD") + b, _ := c.Add("b", "HEAD") + if err := c.Remove(a); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(a.Dir); !os.IsNotExist(err) { + t.Fatal("Remove left the copy behind") + } + c.Cleanup() + if _, err := os.Stat(b.Dir); !os.IsNotExist(err) { + t.Fatal("Cleanup left a copy behind") + } + if _, err := os.Stat(c.root); !os.IsNotExist(err) { + t.Fatal("Cleanup left the root behind") + } +} + +func TestNewPicksBackend(t *testing.T) { + iso, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer iso.Cleanup() + if _, ok := iso.(*CopyIsolator); !ok { + t.Fatalf("plain dir → %T, want *CopyIsolator", iso) + } + repo := initRepo(t) + iso2, err := New(repo) + if err != nil { + t.Fatal(err) + } + defer iso2.Cleanup() + if _, ok := iso2.(*Manager); !ok { + t.Fatalf("repo → %T, want *Manager", iso2) + } +} diff --git a/internal/worktree/isolator.go b/internal/worktree/isolator.go new file mode 100644 index 0000000..4694d6e --- /dev/null +++ b/internal/worktree/isolator.go @@ -0,0 +1,42 @@ +package worktree + +import "github.com/crossben/orchestra-code/internal/gitutil" + +// Isolator gives each concurrent task its own private working directory, shows +// what the task changed, and folds accepted results back into the source +// directory with conflict detection. +// +// Two implementations exist: *Manager (git worktrees, used inside a repository) +// and *CopyIsolator (plain directory copies, used everywhere else). New picks +// the right one. +type Isolator interface { + // Add creates an isolated tree for task id. fromRef names the git ref to + // branch from; the copy isolator ignores it and copies the source's + // current state. + Add(id, fromRef string) (Tree, error) + // Diff returns the tree's own changes as a git-style unified diff. + Diff(t Tree) (string, error) + // DiffStat summarises Diff: files changed and lines added/removed. + DiffStat(t Tree) (files, added, removed int, err error) + // Merge folds the tree's changes into the source. conflict=true means + // nothing was merged and the source is left as it was. + Merge(t Tree, message string) (conflict bool, err error) + // Remove discards one tree. + Remove(t Tree) error + // Cleanup discards every tree created, including ones not yet Removed. + Cleanup() +} + +var ( + _ Isolator = (*Manager)(nil) + _ Isolator = (*CopyIsolator)(nil) +) + +// New returns the isolator for dir: git worktrees inside a repository, plain +// directory copies otherwise. +func New(dir string) (Isolator, error) { + if gitutil.IsRepo(dir) { + return NewManager(dir) + } + return NewCopy(dir) +} From 7741ecf1681838705b1e9bf58fbc3fe062d1e58f Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 19:59:29 +0000 Subject: [PATCH 07/25] router: history-aware agent resolution Resolution order is now: the AI's healthy suggestion -> the agent with the best track record in this directory -> static routes -> default agent -> first healthy agent. The router defines a History interface (BestAgent(dir, candidates)) set via Router.WithHistory; nil keeps the old behaviour. memory.Store implements it: successes = accepted runs + benchmark wins, samples = all runs + wins + validation-failed benchmarks, score = (s+1)/(n+2). A pick needs >=3 samples, more successes than failures and a strictly higher score than every other healthy candidate; ties yield no pick. The decision reason explains it, e.g. "claude: 8/10 accepted in this dir". Wired into the shell and dashboard using their existing memory store; skipped silently when the store did not open. Only the choice of agent changes - dispatch, validation, retry and diff review are untouched. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- cmd/orchestra/context.go | 13 +++ cmd/orchestra/dashboard.go | 2 +- cmd/orchestra/root.go | 2 +- internal/memory/scores.go | 117 +++++++++++++++++++++++++++ internal/memory/scores_test.go | 135 +++++++++++++++++++++++++++++++ internal/router/router.go | 79 ++++++++++++++---- internal/router/router_test.go | 142 +++++++++++++++++++++++++++++++++ 8 files changed, 476 insertions(+), 18 deletions(-) create mode 100644 internal/memory/scores.go create mode 100644 internal/memory/scores_test.go create mode 100644 internal/router/router_test.go diff --git a/README.md b/README.md index 7883d52..0117f03 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ orchestra (claude) › /exit ``` The **AI router** is on by default: plain questions are answered inline, coding tasks auto-route to the -best agent (with a printed reason). Overrides: `@ ` forces an agent; `/route off` switches to +best agent (with a printed reason). The agent is chosen by: the AI's own suggestion → the agent with the +best track record in this directory (accepted runs + benchmark wins, once there are ≥3 outcomes; e.g. +`claude: 8/10 accepted in this dir`) → `router.routes` → `default_agent`. Overrides: `@ ` forces an agent; `/route off` switches to a fixed active agent (`/agent `). Shell commands: `/agents`, `/route [on|off]`, `/agent`, `/help`, `/exit`. Each accepted turn is committed, so the tree stays clean and every turn's diff shows only its own changes. diff --git a/cmd/orchestra/context.go b/cmd/orchestra/context.go index f36d8d2..7d7bf11 100644 --- a/cmd/orchestra/context.go +++ b/cmd/orchestra/context.go @@ -7,6 +7,7 @@ import ( "github.com/crossben/orchestra-code/internal/config" "github.com/crossben/orchestra-code/internal/memory" + "github.com/crossben/orchestra-code/internal/router" "github.com/crossben/orchestra-code/internal/ui" "github.com/crossben/orchestra-code/internal/validate" ) @@ -31,6 +32,18 @@ func absDir() (string, error) { return filepath.Abs(flagDir) } +// The memory store is the router's track-record source. +var _ router.History = (*memory.Store)(nil) + +// withHistory makes r history-aware when the memory store opened. A nil store +// is skipped (not wrapped as a typed-nil interface), keeping routing as before. +func withHistory(r *router.Router, mem *memory.Store) *router.Router { + if mem == nil { + return r + } + return r.WithHistory(mem) +} + // openMemory opens the shared memory store (~/.orchestra/orchestra.db). func openMemory() (*memory.Store, error) { path, err := memory.DefaultPath() diff --git a/cmd/orchestra/dashboard.go b/cmd/orchestra/dashboard.go index eb72d49..99ed118 100644 --- a/cmd/orchestra/dashboard.go +++ b/cmd/orchestra/dashboard.go @@ -43,7 +43,7 @@ func newDashboardCmd() *cobra.Command { fmt.Printf("(note: AI routing off in chat: %v)\n", rerr) routingOn = false } else { - rtr = r + rtr = withHistory(r, mem) } } diff --git a/cmd/orchestra/root.go b/cmd/orchestra/root.go index 9cfcace..2af39a0 100644 --- a/cmd/orchestra/root.go +++ b/cmd/orchestra/root.go @@ -91,7 +91,7 @@ func runShell(cmd *cobra.Command) error { fmt.Printf("(warning: AI routing disabled: %v)\n", rerr) routingOn = false } else { - rtr = r + rtr = withHistory(r, mem) } } diff --git a/internal/memory/scores.go b/internal/memory/scores.go new file mode 100644 index 0000000..7750837 --- /dev/null +++ b/internal/memory/scores.go @@ -0,0 +1,117 @@ +package memory + +import ( + "fmt" + "path/filepath" +) + +// MinHistorySamples is how many outcomes an agent needs in a directory before +// its track record is trusted for routing. +const MinHistorySamples = 3 + +// AgentScore is one agent's track record in a directory, used to route tasks. +// +// Successes = accepted runs + benchmark wins. +// Samples = all runs + benchmark wins + validation-failed benchmarks. +// A benchmark that validated but lost, or skipped validation, is neutral. +type AgentScore struct { + Successes int + Samples int + Benchmarks int // how many of Samples came from benchmarks +} + +// Score is the Laplace-smoothed acceptance rate (successes+1)/(samples+2). +func (a AgentScore) Score() float64 { + return float64(a.Successes+1) / float64(a.Samples+2) +} + +// better reports whether a scores strictly higher than b, compared exactly as +// fractions so ties are never broken by floating-point noise. +func (a AgentScore) better(b AgentScore) bool { + return (a.Successes+1)*(b.Samples+2) > (b.Successes+1)*(a.Samples+2) +} + +// AgentScores returns every agent's track record for dir (exact match). +func (s *Store) AgentScores(dir string) (map[string]AgentScore, error) { + out := map[string]AgentScore{} + rows, err := s.db.Query(`SELECT agent, COUNT(*), SUM(outcome = 'accepted') FROM runs WHERE dir = ? GROUP BY agent`, dir) + if err != nil { + return nil, fmt.Errorf("agent scores: %w", err) + } + for rows.Next() { + var name string + var n, acc int + if err := rows.Scan(&name, &n, &acc); err != nil { + rows.Close() + return nil, err + } + sc := out[name] + sc.Successes += acc + sc.Samples += n + out[name] = sc + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + rows, err = s.db.Query( + `SELECT agent, SUM(won = 1), SUM(won = 0 AND valid = 0 AND skipped = 0) FROM benchmarks WHERE dir = ? GROUP BY agent`, dir) + if err != nil { + return nil, fmt.Errorf("agent scores: %w", err) + } + defer rows.Close() + for rows.Next() { + var name string + var wins, fails int + if err := rows.Scan(&name, &wins, &fails); err != nil { + return nil, err + } + sc := out[name] + sc.Successes += wins + sc.Samples += wins + fails + sc.Benchmarks += wins + fails + out[name] = sc + } + return out, rows.Err() +} + +// BestAgent picks the candidate with the best track record in dir. It only +// picks when an agent has at least MinHistorySamples outcomes, more successes +// than failures, and a strictly higher score than every other qualifying +// candidate; a tie or any error yields ok=false. The reason explains the pick, +// e.g. "claude: 8/10 accepted in this dir". Safe on a nil Store. +func (s *Store) BestAgent(dir string, candidates []string) (name, reason string, ok bool) { + if s == nil || len(candidates) == 0 { + return "", "", false + } + if abs, err := filepath.Abs(dir); err == nil { + dir = abs // runs are recorded under the absolute directory + } + scores, err := s.AgentScores(dir) + if err != nil { + return "", "", false + } + var best AgentScore + tied := false + for _, c := range candidates { + sc, found := scores[c] + if !found || sc.Samples < MinHistorySamples || 2*sc.Successes <= sc.Samples { + continue + } + switch { + case name == "" || sc.better(best): + name, best, tied = c, sc, false + case !best.better(sc): + tied = true + } + } + if name == "" || tied { + return "", "", false + } + reason = fmt.Sprintf("%s: %d/%d accepted in this dir", name, best.Successes, best.Samples) + if best.Benchmarks > 0 { + reason += " (runs + benchmarks)" + } + return name, reason, true +} diff --git a/internal/memory/scores_test.go b/internal/memory/scores_test.go new file mode 100644 index 0000000..38abf92 --- /dev/null +++ b/internal/memory/scores_test.go @@ -0,0 +1,135 @@ +package memory + +import ( + "path/filepath" + "testing" + "time" +) + +func openTemp(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "o.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +// runs records n runs for agent in dir, the first `accepted` of them accepted. +func runs(t *testing.T, s *Store, dir, agent string, accepted, n int) { + t.Helper() + for i := 0; i < n; i++ { + outcome := "rejected" + if i < accepted { + outcome = "accepted" + } + if err := s.Record(Run{Dir: dir, Agent: agent, Prompt: "p", Outcome: outcome, Attempts: 1}, time.Now()); err != nil { + t.Fatal(err) + } + } +} + +func bench(t *testing.T, s *Store, dir, agent string, valid, skipped, won bool) { + t.Helper() + if err := s.RecordBenchmark(BenchRun{Dir: dir, Task: "t", Agent: agent, Valid: valid, Skipped: skipped, Won: won}, time.Now()); err != nil { + t.Fatal(err) + } +} + +func TestAgentScoresCombinesRunsAndBenchmarks(t *testing.T) { + s := openTemp(t) + runs(t, s, "/p", "claude", 2, 3) + bench(t, s, "/p", "claude", true, false, true) // win → success + bench(t, s, "/p", "claude", false, false, false) // validation failed → reject + bench(t, s, "/p", "claude", true, false, false) // valid but lost → neutral + bench(t, s, "/p", "claude", false, true, false) // validation skipped → neutral + runs(t, s, "/other", "claude", 5, 5) // different dir: ignored + + got, err := s.AgentScores("/p") + if err != nil { + t.Fatal(err) + } + c := got["claude"] + if c.Successes != 3 || c.Samples != 5 { + t.Fatalf("claude = %+v, want 3/5", c) + } + if want := 4.0 / 7.0; c.Score() != want { + t.Fatalf("score = %v, want %v", c.Score(), want) + } +} + +func TestBestAgent(t *testing.T) { + all := []string{"claude", "opencode", "mimo"} + cases := []struct { + name string + setup func(*Store) + candidates []string + want string + wantReason string + wantOK bool + }{ + {name: "no history", setup: func(*Store) {}, candidates: all}, + {name: "below min samples", setup: func(s *Store) { runs(t, s, "/p", "claude", 2, 2) }, candidates: all}, + { + name: "clear winner", + setup: func(s *Store) { + runs(t, s, "/p", "claude", 8, 10) + runs(t, s, "/p", "opencode", 1, 3) + }, + candidates: all, want: "claude", wantReason: "claude: 8/10 accepted in this dir", wantOK: true, + }, + { + name: "only candidates considered", + setup: func(s *Store) { + runs(t, s, "/p", "claude", 8, 10) + runs(t, s, "/p", "opencode", 2, 3) + }, + candidates: []string{"opencode", "mimo"}, want: "opencode", wantReason: "opencode: 2/3 accepted in this dir", wantOK: true, + }, + { + name: "tie means no pick", + setup: func(s *Store) { + runs(t, s, "/p", "claude", 3, 3) + runs(t, s, "/p", "opencode", 3, 3) + }, + candidates: all, + }, + { + name: "poor record is never pushed", + setup: func(s *Store) { runs(t, s, "/p", "claude", 1, 4) }, + candidates: all, + }, + { + name: "benchmark wins count", + setup: func(s *Store) { + runs(t, s, "/p", "mimo", 1, 1) + bench(t, s, "/p", "mimo", true, false, true) + bench(t, s, "/p", "mimo", true, false, true) + }, + candidates: all, want: "mimo", wantReason: "mimo: 3/3 accepted in this dir (runs + benchmarks)", wantOK: true, + }, + { + name: "other dir ignored", + setup: func(s *Store) { runs(t, s, "/elsewhere", "claude", 5, 5) }, + candidates: all, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := openTemp(t) + tc.setup(s) + name, reason, ok := s.BestAgent("/p", tc.candidates) + if ok != tc.wantOK || name != tc.want || reason != tc.wantReason { + t.Fatalf("BestAgent = (%q, %q, %v), want (%q, %q, %v)", name, reason, ok, tc.want, tc.wantReason, tc.wantOK) + } + }) + } +} + +func TestBestAgentNilStore(t *testing.T) { + var s *Store + if _, _, ok := s.BestAgent("/p", []string{"claude"}); ok { + t.Fatal("nil store must not pick") + } +} diff --git a/internal/router/router.go b/internal/router/router.go index ac0e981..1a7d32d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -6,8 +6,9 @@ // an agent in query mode; APIClassifier calls a hosted LLM directly over HTTP. // Neither affects the resolution logic here. // -// Resolution is three-tier and never blocks: AI suggestion → static routes → -// default agent. Only healthy (installed) agents are chosen. +// Resolution never blocks: AI suggestion → track record in this directory (when +// a History is wired) → static routes → default agent → first healthy agent. +// Only healthy (installed) agents are chosen. package router import ( @@ -50,6 +51,14 @@ type Classifier interface { Classify(ctx context.Context, message, dir string) (Classification, error) } +// History reports which agent has the best track record in a directory. The +// router uses it as a tie-breaker when the AI makes no usable suggestion. +// Implementations must only return one of candidates, and ok=false when the +// data is too thin or tied. reason explains the pick for the user. +type History interface { + BestAgent(dir string, candidates []string) (name, reason string, ok bool) +} + // Router combines a classifier with agent resolution and direct answering. type Router struct { cls Classifier @@ -57,6 +66,13 @@ type Router struct { reg *agent.Registry // to check agent availability routes map[string]string // intent → agent name (static fallback) fallback string // default agent + history History // optional track-record tie-breaker +} + +// WithHistory enables history-aware resolution (nil disables it) and returns r. +func (r *Router) WithHistory(h History) *Router { + r.history = h + return r } // New builds a Router. @@ -72,10 +88,11 @@ func (r *Router) Classifier() Classifier { return r.cls } func (r *Router) Route(ctx context.Context, message, dir string) Decision { c, err := r.cls.Classify(ctx, message, dir) if err != nil { + name, why := r.resolve(IntentImplement, "", dir) return Decision{ Intent: IntentImplement, - Agent: r.resolve(IntentImplement, ""), - Reason: fmt.Sprintf("classification failed (%v); defaulting to implement", err), + Agent: name, + Reason: joinReason(fmt.Sprintf("classification failed (%v); defaulting to implement", err), why), } } if c.Intent == IntentQuestion { @@ -84,10 +101,23 @@ func (r *Router) Route(ctx context.Context, message, dir string) Decision { if !validIntent(c.Intent) { c.Intent = IntentImplement } + name, why := r.resolve(c.Intent, c.Agent, dir) return Decision{ Intent: c.Intent, - Agent: r.resolve(c.Intent, c.Agent), - Reason: c.Reason, + Agent: name, + Reason: joinReason(c.Reason, why), + } +} + +// joinReason appends a data-driven explanation to the classifier's reason. +func joinReason(base, history string) string { + switch { + case history == "": + return base + case base == "": + return "history: " + history + default: + return base + " · history: " + history } } @@ -104,23 +134,42 @@ func (r *Router) Answer(ctx context.Context, message, dir string, timeout time.D return r.answerer.Query(ctx, task) } -// resolve applies the three-tier fallback, choosing only healthy agents. -func (r *Router) resolve(intent Intent, suggested string) string { +// resolve applies the tiered fallback, choosing only healthy agents. The second +// return value explains a history-driven pick ("" otherwise). +func (r *Router) resolve(intent Intent, suggested, dir string) (string, string) { if r.healthy(suggested) { - return suggested + return suggested, "" + } + var healthy []string + for _, n := range r.reg.Names() { + if r.healthy(n) { + healthy = append(healthy, n) + } + } + if r.history != nil && len(healthy) > 0 { + if name, why, ok := r.history.BestAgent(dir, healthy); ok && contains(healthy, name) { + return name, why + } } if a := r.routes[string(intent)]; r.healthy(a) { - return a + return a, "" } if r.healthy(r.fallback) { - return r.fallback + return r.fallback, "" } - for _, n := range r.reg.Names() { - if r.healthy(n) { - return n + if len(healthy) > 0 { + return healthy[0], "" + } + return r.fallback, "" // nothing healthy; dispatch will surface a clear error +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true } } - return r.fallback // nothing healthy; dispatch will surface a clear error + return false } func (r *Router) healthy(name string) bool { diff --git a/internal/router/router_test.go b/internal/router/router_test.go new file mode 100644 index 0000000..bf320b3 --- /dev/null +++ b/internal/router/router_test.go @@ -0,0 +1,142 @@ +package router + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/crossben/orchestra-code/internal/agent" +) + +type fakeClassifier struct { + c Classification + err error +} + +func (f fakeClassifier) Classify(context.Context, string, string) (Classification, error) { + return f.c, f.err +} + +type fakeHistory struct { + name, reason string + ok bool + gotDir string + gotCands []string + calls int +} + +func (f *fakeHistory) BestAgent(dir string, candidates []string) (string, string, bool) { + f.calls++ + f.gotDir, f.gotCands = dir, candidates + return f.name, f.reason, f.ok +} + +// testRegistry: alpha, beta, gamma are healthy ("true" is on PATH); dead is not. +func testRegistry() *agent.Registry { + reg := agent.NewRegistry() + caps := []agent.Capability{agent.CapImplement} + reg.Add(agent.New("alpha", "true", nil, "", caps)) + reg.Add(agent.New("dead", "orchestra-no-such-binary-xyz", nil, "", caps)) + reg.Add(agent.New("beta", "true", nil, "", caps)) + reg.Add(agent.New("gamma", "true", nil, "", caps)) + return reg +} + +func TestRouteResolutionOrder(t *testing.T) { + routes := map[string]string{"implement": "beta"} + cases := []struct { + name string + cls fakeClassifier + hist *fakeHistory // nil → no history wired + wantAgent string + wantReason string // substring + wantNoHist bool // history must not be consulted + }{ + { + name: "healthy AI suggestion beats history", + cls: fakeClassifier{c: Classification{Intent: IntentImplement, Agent: "alpha", Reason: "ai"}}, + hist: &fakeHistory{name: "gamma", reason: "gamma: 5/6 accepted in this dir", ok: true}, + wantAgent: "alpha", wantReason: "ai", wantNoHist: true, + }, + { + name: "unhealthy suggestion falls to history", + cls: fakeClassifier{c: Classification{Intent: IntentImplement, Agent: "dead", Reason: "ai"}}, + hist: &fakeHistory{name: "gamma", reason: "gamma: 5/6 accepted in this dir", ok: true}, + wantAgent: "gamma", wantReason: "ai · history: gamma: 5/6 accepted in this dir", + }, + { + name: "history beats static route", + cls: fakeClassifier{c: Classification{Intent: IntentImplement}}, + hist: &fakeHistory{name: "alpha", reason: "alpha: 3/3 accepted in this dir", ok: true}, + wantAgent: "alpha", wantReason: "history: alpha: 3/3 accepted in this dir", + }, + { + name: "no history pick uses static route", + cls: fakeClassifier{c: Classification{Intent: IntentImplement, Reason: "ai"}}, + hist: &fakeHistory{}, + wantAgent: "beta", wantReason: "ai", + }, + { + name: "unhealthy history pick is ignored", + cls: fakeClassifier{c: Classification{Intent: IntentImplement}}, + hist: &fakeHistory{name: "dead", reason: "dead: 9/9", ok: true}, + wantAgent: "beta", + }, + { + name: "nil history keeps previous behaviour", + cls: fakeClassifier{c: Classification{Intent: IntentImplement}}, + wantAgent: "beta", + }, + { + name: "no route falls to default agent", + cls: fakeClassifier{c: Classification{Intent: IntentReview}}, + hist: &fakeHistory{}, + wantAgent: "gamma", + }, + { + name: "classification failure still consults history", + cls: fakeClassifier{err: errors.New("boom")}, + hist: &fakeHistory{name: "alpha", reason: "alpha: 4/5 accepted in this dir", ok: true}, + wantAgent: "alpha", wantReason: "history: alpha: 4/5 accepted in this dir", + }, + { + name: "questions never consult history", + cls: fakeClassifier{c: Classification{Intent: IntentQuestion, Reason: "asks"}}, + hist: &fakeHistory{name: "alpha", ok: true}, + wantAgent: "", + wantReason: "asks", wantNoHist: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := New(tc.cls, nil, testRegistry(), routes, "gamma") + if tc.hist != nil { + r = r.WithHistory(tc.hist) + } + d := r.Route(context.Background(), "do it", "/proj") + if d.Agent != tc.wantAgent { + t.Fatalf("agent = %q, want %q (reason %q)", d.Agent, tc.wantAgent, d.Reason) + } + if !strings.Contains(d.Reason, tc.wantReason) { + t.Fatalf("reason = %q, want it to contain %q", d.Reason, tc.wantReason) + } + if tc.hist == nil { + return + } + if tc.wantNoHist { + if tc.hist.calls != 0 { + t.Fatalf("history consulted %d times, want 0", tc.hist.calls) + } + return + } + if tc.hist.calls != 1 || tc.hist.gotDir != "/proj" { + t.Fatalf("history calls=%d dir=%q", tc.hist.calls, tc.hist.gotDir) + } + if want := []string{"alpha", "beta", "gamma"}; !reflect.DeepEqual(tc.hist.gotCands, want) { + t.Fatalf("candidates = %v, want only healthy %v", tc.hist.gotCands, want) + } + }) + } +} From 5b742dd8140661439a8540100dc8a78c357edd6a Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:00:07 +0000 Subject: [PATCH 08/25] llm/agent/config: report token usage and config-priced cost for API agents Providers parse usage (OpenAI prompt/completion_tokens, Anthropic input/output_tokens); a missing usage object is zero, never an error. agent.Result gains Usage; APIAgent fills it and prices it from the new optional price_input_per_mtok / price_output_per_mtok agent fields. Includes the design spec. Co-Authored-By: Claude Opus 5.5 --- .../specs/2026-09-26-usage-cost-design.md | 84 +++++++++++++ internal/agent/agent.go | 1 + internal/agent/apiagent.go | 44 +++++-- internal/agent/usage.go | 71 +++++++++++ internal/agent/usage_test.go | 112 ++++++++++++++++++ internal/config/config.go | 7 ++ internal/config/config_test.go | 26 ++++ internal/llm/anthropic.go | 12 +- internal/llm/llm.go | 8 ++ internal/llm/llm_test.go | 49 ++++++++ internal/llm/openai.go | 5 + 11 files changed, 406 insertions(+), 13 deletions(-) create mode 100644 docs/superpowers/specs/2026-09-26-usage-cost-design.md create mode 100644 internal/agent/usage.go create mode 100644 internal/agent/usage_test.go diff --git a/docs/superpowers/specs/2026-09-26-usage-cost-design.md b/docs/superpowers/specs/2026-09-26-usage-cost-design.md new file mode 100644 index 0000000..4520606 --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-usage-cost-design.md @@ -0,0 +1,84 @@ +# Token & cost tracking (API agents first) + +Date: 2026-09-26 · Status: implemented + +## Goal + +Show how many tokens a run consumed and, when the user has told us the price, +what it cost — per run, per benchmark entry and per agent. API agents report +usage exactly; CLI agents stay "unknown" for now (no scraping of CLI output). + +## Design + +**llm.** `llm.Response` gains `Usage{InputTokens, OutputTokens int}`. +OpenAI-compatible replies are read from `usage.prompt_tokens` / +`usage.completion_tokens`; Anthropic from `usage.input_tokens` / +`usage.output_tokens`. A reply without a `usage` object yields zeros — never an +error (many local gateways omit it). + +**agent.** `agent.Result` gains `Usage agent.Usage`: + +```go +type Usage struct { + InputTokens, OutputTokens int + CostUSD float64 + Known bool // tokens were reported + Priced bool // CostUSD was computed from configured prices +} +``` + +`Usage.Add` sums two usages (Known/Priced are OR-ed). `agent.Cost(in, out, +priceIn, priceOut)` computes USD from per-million-token prices and reports +whether both prices were set (> 0). `APIAgent.SetPricing` stores the prices; +`APIAgent.Run` fills `Result.Usage` from the provider reply (also on an +apply-failure result, since the tokens were spent). CLI agents leave it zero. + +**Pricing is config-only** — no built-in price table (prices go stale): + +```yaml +agents: + - name: sonnet-api + type: api + provider: anthropic + model: claude-sonnet-4-5 + price_input_per_mtok: 3 # USD per 1M input tokens + price_output_per_mtok: 15 # USD per 1M output tokens +``` + +Cost is computed only when both prices are set; otherwise tokens are still +recorded and cost renders as "—". + +**engine.** `Outcome.Usage` and `Turn.Usage` hold the sum across every attempt +(first try + self-correction retries), including an attempt whose agent call +errored. `EventAgentDone` carries that attempt's `Usage`, so the dashboard's +live panel can keep a running total. `run`/`do`/shell print one summary line +after the loop, only when usage is known: +`▸ tokens 12.3k in / 1.1k out · $0.04`. + +**memory.** `runs` and `benchmarks` gain `tokens_in INTEGER NOT NULL DEFAULT 0`, +`tokens_out INTEGER NOT NULL DEFAULT 0`, `cost_usd REAL NOT NULL DEFAULT 0`, +added with the existing idempotent `hasColumn` + `ALTER TABLE ADD COLUMN` +pattern, so old databases upgrade in place. `Run`, `BenchRun`, `BenchRow` and +`AgentStats` carry the values; `StatsByAgent` sums them per agent. In storage, +zero tokens means "unknown" and zero cost means "not priced". + +**display.** Compact cells: tokens as `12.3k/1.1k`, cost as `$0.04` (four +decimals under a cent), "—" when unknown. + +- `orchestra history`: TOKENS and COST columns. +- `orchestra benchmark` leaderboard: TOKENS and COST columns. +- TUI History, Benchmarks: TOKENS (10) and COST (7) columns. +- TUI Agents: per-agent TOKENS and COST totals. +- TUI live run panel: running total under the timeline once known. + +## Non-goals + +Parsing CLI agent output for usage; a built-in price table; cache-token +pricing; tracking router/planner query usage (not part of a run). + +## Tests + +httptest fixtures for both providers with and without `usage`; cost math; +engine summing across a retry with a fake agent; memory migration from an +old-schema DB (twice, idempotent) and round trip; headless TUI renders of the +new columns. diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 7169aa9..d20ad12 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -46,6 +46,7 @@ type Result struct { ExitCode int Duration time.Duration Output string // combined stdout+stderr the agent produced (for question detection) + Usage Usage // tokens/cost when the agent reports them (API agents); zero otherwise } // Agent is anything Orchestra can dispatch a Task to. diff --git a/internal/agent/apiagent.go b/internal/agent/apiagent.go index b361bd2..df5348f 100644 --- a/internal/agent/apiagent.go +++ b/internal/agent/apiagent.go @@ -29,6 +29,8 @@ type APIAgent struct { prov llm.Provider caps []Capability budget int + + priceIn, priceOut float64 // USD per million tokens (0 = not priced) } // Compile-time proof that APIAgent satisfies the whole contract surface, so @@ -67,6 +69,16 @@ func NewAPI(name, provider, model, apiBase, keyEnv string, caps []Capability, bu return &APIAgent{name: name, model: model, keyEnv: keyEnv, prov: prov, caps: copied, budget: budget}, nil } +// SetPricing sets the USD prices per million input/output tokens used to +// cost each run. Both must be > 0 for a cost to be computed; otherwise runs +// still report tokens but no cost. +func (a *APIAgent) SetPricing(inPerMTok, outPerMTok float64) { + a.priceIn, a.priceOut = inPerMTok, outPerMTok +} + +// Pricing returns the configured USD prices per million input/output tokens. +func (a *APIAgent) Pricing() (inPerMTok, outPerMTok float64) { return a.priceIn, a.priceOut } + // Name implements Agent. func (a *APIAgent) Name() string { return a.name } @@ -116,14 +128,16 @@ func (a *APIAgent) Run(ctx context.Context, task Task) (Result, error) { } // There is no token stream, so progress is one line out and one line back. progress(task.Output, "→ asking %s (%s) · %d KiB of repo context", a.model, a.prov.Name(), len(snap)>>10) - text, err := a.complete(ctx, outputContract, snap+"\n\n=== TASK ===\n"+task.Prompt, task.Timeout) + resp, err := a.complete(ctx, outputContract, snap+"\n\n=== TASK ===\n"+task.Prompt, task.Timeout) if err != nil { return Result{}, err } + text, usage := resp.Text, a.usage(resp.Usage) p, _ := patch.Extract(text) if !p.Empty() { if err := patch.Apply(ctx, task.Dir, p); err != nil { - return Result{Duration: time.Since(start)}, fmt.Errorf("apply model changes: %w", err) + // The tokens were spent even though the patch failed. + return Result{Duration: time.Since(start), Usage: usage}, fmt.Errorf("apply model changes: %w", err) } progress(task.Output, "← applied %d diff(s) and %d file write(s) in %s", len(p.Diffs), len(p.Files), time.Since(start).Round(time.Second/10)) } else { @@ -131,7 +145,17 @@ func (a *APIAgent) Run(ctx context.Context, task Task) (Result, error) { } // Output carries the model's raw text so the engine's no-change/question // detection (package engine) sees exactly what the model said. - return Result{ExitCode: 0, Duration: time.Since(start), Output: text}, nil + return Result{ExitCode: 0, Duration: time.Since(start), Output: text, Usage: usage}, nil +} + +// usage converts a provider's token counts into a (possibly priced) Usage. +// A reply without counts stays unknown rather than reading as free. +func (a *APIAgent) usage(u llm.Usage) Usage { + if u.InputTokens == 0 && u.OutputTokens == 0 { + return Usage{} + } + cost, priced := Cost(u.InputTokens, u.OutputTokens, a.priceIn, a.priceOut) + return Usage{InputTokens: u.InputTokens, OutputTokens: u.OutputTokens, CostUSD: cost, Known: true, Priced: priced} } // progress writes one line to w when it is set. @@ -149,7 +173,8 @@ func (a *APIAgent) RunQuiet(ctx context.Context, task Task) (Result, error) { // Query implements Querier: a plain completion with no patch contract and no // repo snapshot — used by the router (classification/answers) and planner. func (a *APIAgent) Query(ctx context.Context, task Task) (string, error) { - return a.complete(ctx, "", task.Prompt, task.Timeout) + resp, err := a.complete(ctx, "", task.Prompt, task.Timeout) + return resp.Text, err } // QueryQuiet implements QuietQuerier; identical to Query (never streams). @@ -162,7 +187,8 @@ const probeContract = "Reply with exactly the word OK and nothing else. Do not c // Probe implements Prober: a trivial live completion that catches auth, // billing, and connectivity problems a Health check cannot see. func (a *APIAgent) Probe(ctx context.Context, timeout time.Duration) ProbeResult { - text, err := a.complete(ctx, probeContract, probeContract, timeout) + resp, err := a.complete(ctx, probeContract, probeContract, timeout) + text := resp.Text switch { case err != nil: detail := firstMeaningfulLine(err.Error()) @@ -179,19 +205,15 @@ func (a *APIAgent) Probe(ctx context.Context, timeout time.Duration) ProbeResult } // complete issues one provider call. system may be empty (plain query mode). -func (a *APIAgent) complete(ctx context.Context, system, user string, timeout time.Duration) (string, error) { +func (a *APIAgent) complete(ctx context.Context, system, user string, timeout time.Duration) (llm.Response, error) { if timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } - resp, err := a.prov.Complete(ctx, llm.Request{ + return a.prov.Complete(ctx, llm.Request{ System: system, Messages: []llm.Message{{Role: "user", Content: user}}, Temperature: 0, }) - if err != nil { - return "", err - } - return resp.Text, nil } diff --git a/internal/agent/usage.go b/internal/agent/usage.go new file mode 100644 index 0000000..6b50b89 --- /dev/null +++ b/internal/agent/usage.go @@ -0,0 +1,71 @@ +package agent + +import "fmt" + +// Usage is the token accounting for one agent run (or, summed, for every +// attempt of a supervised run). API agents fill it from the provider reply; +// CLI agents leave it zero — Known false renders as "unknown", not "free". +type Usage struct { + InputTokens int + OutputTokens int + CostUSD float64 + Known bool // the provider reported token counts + Priced bool // CostUSD was computed from configured prices +} + +// Add returns the sum of u and o. Known/Priced are true when either side is. +func (u Usage) Add(o Usage) Usage { + return Usage{ + InputTokens: u.InputTokens + o.InputTokens, + OutputTokens: u.OutputTokens + o.OutputTokens, + CostUSD: u.CostUSD + o.CostUSD, + Known: u.Known || o.Known, + Priced: u.Priced || o.Priced, + } +} + +// String is the one-line run summary, e.g. "tokens 12.3k in / 1.1k out · +// $0.04" — the cost part only when priced. Empty when usage is unknown. +func (u Usage) String() string { + if !u.Known { + return "" + } + s := fmt.Sprintf("tokens %s in / %s out", FormatTokens(u.InputTokens), FormatTokens(u.OutputTokens)) + if u.Priced { + s += " · " + FormatCost(u.CostUSD) + } + return s +} + +// Cost prices a run from per-million-token USD prices. Prices are config-only +// (a built-in table would go stale), so cost is computed only when both are +// set (> 0); priced reports whether it was. +func Cost(inTokens, outTokens int, priceInPerMTok, priceOutPerMTok float64) (usd float64, priced bool) { + if priceInPerMTok <= 0 || priceOutPerMTok <= 0 { + return 0, false + } + return (float64(inTokens)*priceInPerMTok + float64(outTokens)*priceOutPerMTok) / 1e6, true +} + +// FormatTokens renders a token count compactly: 950, 12.3k, 123k, 1.2M. +func FormatTokens(n int) string { + switch { + case n < 1000: + return fmt.Sprint(n) + case n < 100_000: + return fmt.Sprintf("%.1fk", float64(n)/1e3) + case n < 1_000_000: + return fmt.Sprintf("%.0fk", float64(n)/1e3) + default: + return fmt.Sprintf("%.1fM", float64(n)/1e6) + } +} + +// FormatCost renders USD with cents, or four decimals below one cent so a +// cheap run does not read as free. +func FormatCost(usd float64) string { + if usd > 0 && usd < 0.01 { + return fmt.Sprintf("$%.4f", usd) + } + return fmt.Sprintf("$%.2f", usd) +} diff --git a/internal/agent/usage_test.go b/internal/agent/usage_test.go new file mode 100644 index 0000000..7ea44f8 --- /dev/null +++ b/internal/agent/usage_test.go @@ -0,0 +1,112 @@ +package agent + +import ( + "context" + "math" + "testing" + + "github.com/crossben/orchestra-code/internal/llm" +) + +func TestCost(t *testing.T) { + cases := []struct { + name string + in, out int + pIn, pOut float64 + want float64 + wantPriced bool + }{ + {"both prices", 1_000_000, 500_000, 3, 15, 3 + 7.5, true}, + {"small run", 12_300, 1_100, 3, 15, 0.0369 + 0.0165, true}, + {"input price only", 1000, 1000, 3, 0, 0, false}, + {"no prices", 1000, 1000, 0, 0, 0, false}, + {"zero tokens priced", 0, 0, 3, 15, 0, true}, + } + for _, c := range cases { + got, priced := Cost(c.in, c.out, c.pIn, c.pOut) + if priced != c.wantPriced || math.Abs(got-c.want) > 1e-9 { + t.Errorf("%s: Cost = %v,%v want %v,%v", c.name, got, priced, c.want, c.wantPriced) + } + } +} + +func TestUsageAddAndString(t *testing.T) { + var total Usage + if total.String() != "" { + t.Fatalf("unknown usage should render empty, got %q", total.String()) + } + total = total.Add(Usage{InputTokens: 12_000, OutputTokens: 600, CostUSD: 0.03, Known: true, Priced: true}) + total = total.Add(Usage{}) // an attempt with no reported usage changes nothing + total = total.Add(Usage{InputTokens: 300, OutputTokens: 500, CostUSD: 0.01, Known: true, Priced: true}) + if total.InputTokens != 12_300 || total.OutputTokens != 1_100 || !total.Known || !total.Priced { + t.Fatalf("sum = %+v", total) + } + if got, want := total.String(), "tokens 12.3k in / 1.1k out · $0.04"; got != want { + t.Fatalf("String = %q, want %q", got, want) + } + unpriced := Usage{InputTokens: 950, OutputTokens: 40, Known: true} + if got, want := unpriced.String(), "tokens 950 in / 40 out"; got != want { + t.Fatalf("String = %q, want %q", got, want) + } +} + +func TestFormatTokensAndCost(t *testing.T) { + for n, want := range map[int]string{0: "0", 999: "999", 1000: "1.0k", 12_345: "12.3k", 123_456: "123k", 1_234_567: "1.2M"} { + if got := FormatTokens(n); got != want { + t.Errorf("FormatTokens(%d) = %q, want %q", n, got, want) + } + } + for c, want := range map[float64]string{0.04: "$0.04", 1.5: "$1.50", 0.0042: "$0.0042", 0: "$0.00"} { + if got := FormatCost(c); got != want { + t.Errorf("FormatCost(%v) = %q, want %q", c, got, want) + } + } +} + +// usageProvider is a fake llm.Provider that reports token usage. +type usageProvider struct { + fakeProvider + usage llm.Usage +} + +func (u *usageProvider) Complete(ctx context.Context, req llm.Request) (llm.Response, error) { + r, err := u.fakeProvider.Complete(ctx, req) + r.Usage = u.usage + return r, err +} + +func TestAPIAgentRunReportsUsage(t *testing.T) { + dir := apiRepo(t) + p := &usageProvider{fakeProvider: fakeProvider{text: diffReply}, usage: llm.Usage{InputTokens: 2_000_000, OutputTokens: 100_000}} + + a := newTestAPIAgent(t, p) + res, err := a.Run(context.Background(), Task{Prompt: "add a line", Dir: dir}) + if err != nil { + t.Fatal(err) + } + if u := res.Usage; !u.Known || u.Priced || u.InputTokens != 2_000_000 || u.OutputTokens != 100_000 || u.CostUSD != 0 { + t.Fatalf("unpriced usage = %+v", u) + } + + a.SetPricing(3, 15) + p.text = "nothing to change" + res, err = a.Run(context.Background(), Task{Prompt: "prose please", Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if u := res.Usage; !u.Known || !u.Priced || math.Abs(u.CostUSD-7.5) > 1e-9 { + t.Fatalf("priced usage = %+v", u) + } +} + +func TestAPIAgentRunNoUsageIsUnknown(t *testing.T) { + a := newTestAPIAgent(t, &fakeProvider{text: "nothing to do"}) + a.SetPricing(3, 15) + res, err := a.Run(context.Background(), Task{Prompt: "x", Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if res.Usage.Known || res.Usage.Priced { + t.Fatalf("a reply without usage must stay unknown: %+v", res.Usage) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 2bcc18e..a58ad88 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,12 @@ type AgentConfig struct { APIBase string `yaml:"api_base"` // optional endpoint override APIKeyEnv string `yaml:"api_key_env"` // env var holding the key (provider default) ContextBudget int `yaml:"context_budget"` // repo-snapshot byte cap (0 → DefaultContextBudget) + + // Optional pricing (USD per million tokens). There is no built-in price + // table — prices go stale — so cost is computed only when both are set; + // otherwise tokens are still recorded and cost shows as "—". + PriceInputPerMTok float64 `yaml:"price_input_per_mtok"` + PriceOutputPerMTok float64 `yaml:"price_output_per_mtok"` } // ValidateConfig configures the validation pipeline. Any empty stage is skipped. @@ -354,6 +360,7 @@ func (c *Config) BuildRegistry() *agent.Registry { fmt.Printf("(warning: skipping api agent %q: %v)\n", ac.Name, err) continue } + a.SetPricing(ac.PriceInputPerMTok, ac.PriceOutputPerMTok) reg.Add(a) continue } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3227eab..62e442a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -56,6 +56,32 @@ agents: if len(names) < 2 || names[len(names)-1] != "gpt" { t.Fatalf("registry names: %v", names) } + if in, out := api.Pricing(); in != 0 || out != 0 { + t.Fatalf("no price fields → unpriced, got %v/%v", in, out) + } +} + +func TestBuildRegistryAPIAgentPricing(t *testing.T) { + cfg, err := parseForTest(t, ` +agents: + - name: sonnet + type: api + provider: anthropic + model: claude-sonnet + price_input_per_mtok: 3 + price_output_per_mtok: 15.5 +`) + if err != nil { + t.Fatal(err) + } + a, _ := cfg.BuildRegistry().Get("sonnet") + api, ok := a.(*agent.APIAgent) + if !ok { + t.Fatalf("expected *agent.APIAgent, got %T", a) + } + if in, out := api.Pricing(); in != 3 || out != 15.5 { + t.Fatalf("pricing = %v/%v, want 3/15.5", in, out) + } } func TestBuildRegistrySkipsBrokenAPIAgent(t *testing.T) { diff --git a/internal/llm/anthropic.go b/internal/llm/anthropic.go index 1251bad..2266556 100644 --- a/internal/llm/anthropic.go +++ b/internal/llm/anthropic.go @@ -38,7 +38,11 @@ type anthropicResponse struct { Text string `json:"text"` } `json:"content"` StopReason string `json:"stop_reason"` - Error *struct { + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` + Error *struct { Message string `json:"message"` Type string `json:"type"` } `json:"error"` @@ -83,5 +87,9 @@ func (a *Anthropic) Complete(ctx context.Context, req Request) (Response, error) b.WriteString(c.Text) } } - return Response{Text: b.String(), StopReason: out.StopReason}, nil + return Response{ + Text: b.String(), + StopReason: out.StopReason, + Usage: Usage{InputTokens: out.Usage.InputTokens, OutputTokens: out.Usage.OutputTokens}, + }, nil } diff --git a/internal/llm/llm.go b/internal/llm/llm.go index 3da7814..f237672 100644 --- a/internal/llm/llm.go +++ b/internal/llm/llm.go @@ -34,6 +34,14 @@ type Request struct { type Response struct { Text string // assistant text StopReason string // provider-reported stop reason (informational) + Usage Usage // token counts; zero when the provider did not report them +} + +// Usage is the token accounting a provider reports for one completion. Many +// gateways omit it; a missing usage object is zero, never an error. +type Usage struct { + InputTokens int + OutputTokens int } // Provider abstracts one HTTP LLM API. diff --git a/internal/llm/llm_test.go b/internal/llm/llm_test.go index 4f5befa..cffe1c1 100644 --- a/internal/llm/llm_test.go +++ b/internal/llm/llm_test.go @@ -230,3 +230,52 @@ func TestEmptyChoicesIsServerError(t *testing.T) { t.Fatalf("unhelpful message %q; want it to contain %q", le.Error(), want) } } + +// --- usage parsing --- + +// serveJSON returns a fake API server that always replies with body. +func serveJSON(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestProvidersParseUsage(t *testing.T) { + cases := []struct { + name, provider, body string + want Usage + }{ + {"openai with usage", "openai", + `{"choices":[{"message":{"role":"assistant","content":"x"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1200,"completion_tokens":34,"total_tokens":1234}}`, + Usage{InputTokens: 1200, OutputTokens: 34}}, + {"openai without usage", "openai", + `{"choices":[{"message":{"role":"assistant","content":"x"},"finish_reason":"stop"}]}`, + Usage{}}, + {"anthropic with usage", "anthropic", + `{"content":[{"type":"text","text":"x"}],"stop_reason":"end_turn","usage":{"input_tokens":900,"output_tokens":55}}`, + Usage{InputTokens: 900, OutputTokens: 55}}, + {"anthropic without usage", "anthropic", + `{"content":[{"type":"text","text":"x"}],"stop_reason":"end_turn"}`, + Usage{}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + srv := serveJSON(t, c.body) + p, err := New(c.provider, srv.URL, "k", "m", nil) + if err != nil { + t.Fatal(err) + } + resp, err := p.Complete(context.Background(), Request{Messages: []Message{{Role: "user", Content: "hi"}}}) + if err != nil { + t.Fatalf("missing/present usage must never be an error: %v", err) + } + if resp.Text != "x" || resp.Usage != c.want { + t.Fatalf("resp = %+v, want usage %+v", resp, c.want) + } + }) + } +} diff --git a/internal/llm/openai.go b/internal/llm/openai.go index 01b1fb9..b692b61 100644 --- a/internal/llm/openai.go +++ b/internal/llm/openai.go @@ -34,6 +34,10 @@ type openaiResponse struct { Message openaiMessage `json:"message"` FinishReason string `json:"finish_reason"` } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` Error *struct { Message string `json:"message"` Type string `json:"type"` @@ -67,5 +71,6 @@ func (o *OpenAI) Complete(ctx context.Context, req Request) (Response, error) { return Response{ Text: out.Choices[0].Message.Content, StopReason: out.Choices[0].FinishReason, + Usage: Usage{InputTokens: out.Usage.PromptTokens, OutputTokens: out.Usage.CompletionTokens}, }, nil } From fc54617c33947df08866106215291862e78fced3 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:01:21 +0000 Subject: [PATCH 09/25] docs: spec for the self-updater Co-Authored-By: Claude Opus 5.5 --- .../specs/2026-09-26-auto-update-design.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-26-auto-update-design.md diff --git a/docs/superpowers/specs/2026-09-26-auto-update-design.md b/docs/superpowers/specs/2026-09-26-auto-update-design.md new file mode 100644 index 0000000..ef937b2 --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-auto-update-design.md @@ -0,0 +1,81 @@ +# Self-updater — design + +Date: 2026-09-26 · Scope: new `internal/update` package, a launch hook and an `orchestra update` +subcommand in `cmd/orchestra`. No change to the engine, agents or the supervised loop. + +## Intent + +Users who installed a release archive should find out about new releases without watching GitHub, and +be able to upgrade in place with one keypress. After an upgrade the old process exits so the user never +keeps working in a binary that no longer matches what is on disk. + +Non-goals: background/silent updates, updating Homebrew/Scoop installs through their package manager, +signature verification beyond the published `checksums.txt`, elevating privileges (no sudo). + +## Constraints + +- Stdlib only (`net/http`, `archive/tar`, `archive/zip`, `compress/gzip`, `crypto/sha256`). CGO-free. +- Never breaks non-interactive use: no network, no prompt, no output unless stdin **and** stdout are + terminals. Tests stay offline and deterministic (`httptest`, temp dirs). +- Relies on the GoReleaser contract: archive `orchestra___.tar.gz` (`.zip` on + windows) containing `orchestra`/`orchestra.exe`, plus a `checksums.txt` in ` ` format. + +## `internal/update` + +- `Check(ctx, current) (Release, bool, error)` — GET `/repos/crossben/orchestra-code/releases/latest` + (base URL injectable), parse `tag_name`, `prerelease`, `assets[].{name,browser_download_url,size}`. + Returns `newer=true` only when the latest release is strictly greater by semver. + - `Checkable(current)` is false for `dev`, unparseable versions and GoReleaser snapshots (`-next`); + `Check` then returns immediately without touching the network. + - A prerelease (tag suffix or GitHub `prerelease` flag) is never offered over a stable build. A + prerelease build may be offered the matching stable (`0.9.0-rc1` → `0.9.0`) or a newer prerelease. +- `Install(ctx, rel, exePath)`: + 1. Pick the asset for `runtime.GOOS/GOARCH`; error if the release has none for this platform. + 2. Download `checksums.txt` (1 MiB cap) and the archive (256 MiB cap). + 3. Verify SHA-256 — refuse on mismatch or when the archive has no entry. + 4. Extract only the `orchestra`/`orchestra.exe` regular-file entry (at the root or one directory + deep). Any absolute or `..` entry makes the whole archive unsafe → error. + 5. Resolve symlinks of `exePath`; write a temp file next to it, `chmod 0755`, then replace: + unix: `os.Rename` over the exe. windows: rename the running exe to `.old`, move the new one in + (roll back on failure); `CleanupOld` removes a stale `.old` on the next launch. + 6. A non-writable directory yields a clear error suggesting re-running with sufficient permissions or + reinstalling via the original method. +- Throttle state in `~/.orchestra/update.json`: `{last_check, latest}`. `Due(state, now)` is true after + 24h. The launch hook records the attempt time *before* the network call, so a slow or offline network + costs at most one 2s wait per day. + +## Launch hook (`PersistentPreRunE` on root) + +Skipped (silently, no network) when any holds: + +- stdin or stdout is not a terminal (pipes, scripts, tests, TUI-in-CI); +- `CI` is set; `ORCHESTRA_NO_UPDATE_CHECK=1`; +- the version is not checkable (`dev`, unparseable, `-next` snapshot); +- the command is `help`, `completion` (and its children), cobra's `__complete*`, or `update`; +- `--version`/`--help` (cobra handles these before pre-run anyway); +- the last check was less than 24h ago. + +Flow: 2s timeout on the check; errors are silent (printed to stderr only with `ORCHESTRA_DEBUG=1`). If +newer: `Orchestra vX is available (you have vY). Download and install now? [Y/n]`. + +- Yes (Enter/y/yes): install behind a TTY-aware spinner (`internal/ui`), print + `Updated to vX. Please relaunch orchestra.` and `os.Exit(0)` — the old process does not continue. +- No: continue with the requested command. The throttle means no re-prompt for 24h. +- Install failure: print the error, continue running the current version. + +Stdin is read one byte at a time up to the newline so the prompt never swallows input meant for the +shell that starts afterwards. + +## `orchestra update` + +Forces a check (ignores the throttle, still records it). `--check` only reports; `-y/--yes` skips the +prompt. Same install/exit behaviour as the hook. On a dev/snapshot build it explains that updates are +only offered for release builds. + +## Tests + +Version comparison table; `Checkable`; `Check` parsing, non-200, prerelease handling, dev short-circuit +(server must not be hit); throttle round-trip with a temp state path; `Install` happy path with an +in-test tar.gz and zip + checksums; checksum mismatch; missing checksum entry; missing platform asset; +traversal entry; replacement of a temp "exe" (unix rename and windows `.old` strategy); hook skip +conditions (non-TTY, CI, env var, dev version, skipped commands); prompt answer parsing. From 45fd73c14b675b12b98da30fdcb0b3528ece53b7 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:01:21 +0000 Subject: [PATCH 10/25] update: check GitHub Releases and replace the binary in place Check compares the latest release tag by semver (never offering a prerelease to a stable build; dev/snapshot builds skip the network). Install picks the GoReleaser archive for this platform, verifies its SHA-256 against checksums.txt, extracts only the orchestra binary (refusing traversal entries) and atomically swaps the resolved executable (rename-aside to .old on windows). A small JSON state file throttles automatic checks to once per 24h. Co-Authored-By: Claude Opus 5.5 --- internal/update/check.go | 238 ++++++++++++++++ internal/update/install.go | 300 ++++++++++++++++++++ internal/update/update_test.go | 503 +++++++++++++++++++++++++++++++++ internal/update/version.go | 138 +++++++++ 4 files changed, 1179 insertions(+) create mode 100644 internal/update/check.go create mode 100644 internal/update/install.go create mode 100644 internal/update/update_test.go create mode 100644 internal/update/version.go diff --git a/internal/update/check.go b/internal/update/check.go new file mode 100644 index 0000000..e78409e --- /dev/null +++ b/internal/update/check.go @@ -0,0 +1,238 @@ +// Package update checks GitHub Releases for a newer Orchestra and replaces the +// running binary with it. It relies on the GoReleaser contract in +// .goreleaser.yaml: archives named orchestra___.tar.gz +// (.zip on windows) holding the orchestra binary, plus a checksums.txt. +// +// Stdlib only; every network endpoint is injectable so tests run offline. +package update + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +const ( + // DefaultAPIBase is GitHub's REST API root. + DefaultAPIBase = "https://api.github.com" + // DefaultRepo is where Orchestra releases are published. + DefaultRepo = "crossben/orchestra-code" + // CheckInterval throttles automatic checks. + CheckInterval = 24 * time.Hour + + defaultMaxDownload = 256 << 20 // archive cap + maxMetadata = 1 << 20 // API JSON / checksums.txt cap +) + +// Asset is one downloadable file attached to a release. +type Asset struct { + Name string `json:"name"` + URL string `json:"browser_download_url"` + Size int64 `json:"size"` +} + +// Release is the latest published release. +type Release struct { + Tag string // e.g. "v0.9.0" + Version string // Tag without the leading "v" (the GoReleaser .Version) + Prerelease bool // GitHub's prerelease flag + Assets []Asset // archives + checksums.txt +} + +type ghRelease struct { + TagName string `json:"tag_name"` + Prerelease bool `json:"prerelease"` + Assets []Asset `json:"assets"` +} + +// Client talks to the release host. The zero value uses GitHub and the running +// platform; tests override the fields. +type Client struct { + APIBase string // default DefaultAPIBase + Repo string // default DefaultRepo + HTTP *http.Client // default: a client with a 5 minute ceiling + GOOS string // default runtime.GOOS + GOARCH string // default runtime.GOARCH + UserAgent string // default "orchestra-updater" + MaxDownload int64 // archive size cap in bytes; default 256 MiB +} + +func (c *Client) apiBase() string { + if c.APIBase != "" { + return strings.TrimRight(c.APIBase, "/") + } + return DefaultAPIBase +} + +func (c *Client) repo() string { + if c.Repo != "" { + return c.Repo + } + return DefaultRepo +} + +func (c *Client) http() *http.Client { + if c.HTTP != nil { + return c.HTTP + } + return &http.Client{Timeout: 5 * time.Minute} +} + +func (c *Client) goos() string { + if c.GOOS != "" { + return c.GOOS + } + return runtime.GOOS +} + +func (c *Client) goarch() string { + if c.GOARCH != "" { + return c.GOARCH + } + return runtime.GOARCH +} + +func (c *Client) maxDownload() int64 { + if c.MaxDownload > 0 { + return c.MaxDownload + } + return defaultMaxDownload +} + +// get performs a GET and returns the body, capped at limit bytes. +func (c *Client) get(ctx context.Context, url, accept string, limit int64) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + ua := c.UserAgent + if ua == "" { + ua = "orchestra-updater" + } + req.Header.Set("User-Agent", ua) + if accept != "" { + req.Header.Set("Accept", accept) + } + resp, err := c.http().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: %s", url, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > limit { + return nil, fmt.Errorf("GET %s: response exceeds the %d byte limit", url, limit) + } + return body, nil +} + +// Check fetches the latest release and reports whether it is newer than +// current. Uncheckable builds (see Checkable) return immediately without any +// network access. A prerelease is never offered to a stable build. +func (c *Client) Check(ctx context.Context, current string) (Release, bool, error) { + if !Checkable(current) { + return Release{}, false, nil + } + url := fmt.Sprintf("%s/repos/%s/releases/latest", c.apiBase(), c.repo()) + body, err := c.get(ctx, url, "application/vnd.github+json", maxMetadata) + if err != nil { + return Release{}, false, err + } + var gr ghRelease + if err := json.Unmarshal(body, &gr); err != nil { + return Release{}, false, fmt.Errorf("parse release: %w", err) + } + latest, err := parseVersion(gr.TagName) + if err != nil { + return Release{}, false, fmt.Errorf("latest release: %w", err) + } + cur, _ := parseVersion(current) // Checkable guarantees it parses + rel := Release{ + Tag: gr.TagName, + Version: strings.TrimPrefix(gr.TagName, "v"), + Prerelease: gr.Prerelease || latest.prerelease(), + Assets: gr.Assets, + } + if rel.Prerelease && !cur.prerelease() { + return rel, false, nil + } + return rel, latest.compare(cur) > 0, nil +} + +// Check uses a default Client. +func Check(ctx context.Context, current string) (Release, bool, error) { + return (&Client{}).Check(ctx, current) +} + +// ---- throttle state ---- + +// State is persisted between launches to throttle automatic checks. +type State struct { + LastCheck time.Time `json:"last_check"` + Latest string `json:"latest,omitempty"` +} + +// Due reports whether an automatic check should run now. A clock that moved +// backwards does not trigger a check. +func (s State) Due(now time.Time) bool { + if s.LastCheck.IsZero() { + return true + } + return now.Sub(s.LastCheck) >= CheckInterval +} + +// DefaultStatePath is ~/.orchestra/update.json. +func DefaultStatePath() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".orchestra", "update.json"), nil +} + +// LoadState reads the state file; a missing or corrupt file yields the zero +// State (i.e. a check is due). +func LoadState(path string) State { + var s State + b, err := os.ReadFile(path) + if err != nil { + return State{} + } + if json.Unmarshal(b, &s) != nil { + return State{} + } + return s +} + +// SaveState writes the state file, creating its directory. +func SaveState(path string, s State) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// ErrNotWritable is returned (wrapped) when the executable's directory cannot +// be written, e.g. a system-wide install. +var ErrNotWritable = errors.New("install directory is not writable") diff --git a/internal/update/install.go b/internal/update/install.go new file mode 100644 index 0000000..711ae53 --- /dev/null +++ b/internal/update/install.go @@ -0,0 +1,300 @@ +package update + +import ( + "archive/tar" + "archive/zip" + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" +) + +// AssetName is the archive GoReleaser publishes for a platform. It must stay +// in sync with archives.name_template in .goreleaser.yaml. +func AssetName(version, goos, goarch string) string { + ext := "tar.gz" + if goos == "windows" { + ext = "zip" + } + return fmt.Sprintf("orchestra_%s_%s_%s.%s", version, goos, goarch, ext) +} + +const checksumsName = "checksums.txt" + +// Install downloads rel for this platform, verifies it against checksums.txt, +// and atomically replaces the executable at exePath (symlinks are resolved, so +// the real file is replaced and the link is left alone). On any error the +// existing executable is left untouched. +func (c *Client) Install(ctx context.Context, rel Release, exePath string) error { + goos, goarch := c.goos(), c.goarch() + name := AssetName(rel.Version, goos, goarch) + archive, ok := findAsset(rel.Assets, name) + if !ok { + return fmt.Errorf("release %s has no build for %s/%s (expected %s)", rel.Tag, goos, goarch, name) + } + sums, ok := findAsset(rel.Assets, checksumsName) + if !ok { + return fmt.Errorf("release %s has no %s; refusing to install an unverified binary", rel.Tag, checksumsName) + } + + target, err := filepath.EvalSymlinks(exePath) + if err != nil { + return fmt.Errorf("locate executable: %w", err) + } + if err := checkWritable(filepath.Dir(target)); err != nil { + return err + } + + sumBody, err := c.get(ctx, sums.URL, "", maxMetadata) + if err != nil { + return fmt.Errorf("download %s: %w", checksumsName, err) + } + want, ok := parseChecksums(sumBody)[name] + if !ok { + return fmt.Errorf("%s has no checksum entry for %s; refusing to install", checksumsName, name) + } + + data, err := c.get(ctx, archive.URL, "application/octet-stream", c.maxDownload()) + if err != nil { + return fmt.Errorf("download %s: %w", name, err) + } + got := sha256.Sum256(data) + if !strings.EqualFold(hex.EncodeToString(got[:]), want) { + return fmt.Errorf("checksum mismatch for %s (got %x, want %s); refusing to install", name, got, want) + } + + binName := "orchestra" + if goos == "windows" { + binName = "orchestra.exe" + } + var bin []byte + if strings.HasSuffix(name, ".zip") { + bin, err = extractZip(data, binName, c.maxDownload()) + } else { + bin, err = extractTarGz(data, binName, c.maxDownload()) + } + if err != nil { + return err + } + return replaceExecutable(target, bin, goos == "windows") +} + +// Install uses a default Client. +func Install(ctx context.Context, rel Release, exePath string) error { + return (&Client{}).Install(ctx, rel, exePath) +} + +func findAsset(assets []Asset, name string) (Asset, bool) { + for _, a := range assets { + if a.Name == name { + return a, true + } + } + return Asset{}, false +} + +// parseChecksums reads GoReleaser's " " lines. +func parseChecksums(b []byte) map[string]string { + out := map[string]string{} + sc := bufio.NewScanner(bytes.NewReader(b)) + for sc.Scan() { + f := strings.Fields(sc.Text()) + if len(f) != 2 || len(f[0]) != 64 { + continue + } + out[strings.TrimPrefix(f[1], "*")] = strings.ToLower(f[0]) + } + return out +} + +// entryMatches validates an archive entry name and reports whether it is the +// binary: either at the archive root or one directory deep. Any absolute or +// parent-escaping entry makes the whole archive unsafe. +func entryMatches(name, binName string) (bool, error) { + n := strings.ReplaceAll(name, "\\", "/") + if strings.HasPrefix(n, "/") || filepath.IsAbs(name) || (len(n) > 1 && n[1] == ':') { + return false, fmt.Errorf("archive contains unsafe path %q", name) + } + for _, part := range strings.Split(n, "/") { + if part == ".." { + return false, fmt.Errorf("archive contains unsafe path %q", name) + } + } + clean := path.Clean(n) + if path.Base(clean) != binName { + return false, nil + } + return strings.Count(clean, "/") <= 1, nil +} + +func readCapped(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > limit { + return nil, fmt.Errorf("binary in archive exceeds the %d byte limit", limit) + } + return b, nil +} + +func extractTarGz(data []byte, binName string, limit int64) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("open archive: %w", err) + } + defer gz.Close() + tr := tar.NewReader(gz) + var bin []byte + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read archive: %w", err) + } + ok, err := entryMatches(hdr.Name, binName) + if err != nil { + return nil, err + } + if !ok || hdr.Typeflag != tar.TypeReg || bin != nil { + continue + } + if bin, err = readCapped(tr, limit); err != nil { + return nil, err + } + } + if len(bin) == 0 { + return nil, fmt.Errorf("archive does not contain %s", binName) + } + return bin, nil +} + +func extractZip(data []byte, binName string, limit int64) ([]byte, error) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open archive: %w", err) + } + var bin []byte + for _, f := range zr.File { + ok, err := entryMatches(f.Name, binName) + if err != nil { + return nil, err + } + if !ok || !f.Mode().IsRegular() || bin != nil { + continue + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("read archive: %w", err) + } + bin, err = readCapped(rc, limit) + rc.Close() + if err != nil { + return nil, err + } + } + if len(bin) == 0 { + return nil, fmt.Errorf("archive does not contain %s", binName) + } + return bin, nil +} + +func notWritable(dir string, err error) error { + return fmt.Errorf("%w: cannot write to %s (%v) — re-run `orchestra update` as a user who can write there, or reinstall the new release the way you installed Orchestra (e.g. brew/scoop)", ErrNotWritable, dir, err) +} + +// checkWritable probes dir before downloading anything, so a system-wide +// install fails fast with a clear hint. +func checkWritable(dir string) error { + f, err := os.CreateTemp(dir, ".orchestra-probe-*") + if err != nil { + if errors.Is(err, fs.ErrPermission) { + return notWritable(dir, err) + } + return fmt.Errorf("prepare update in %s: %w", dir, err) + } + name := f.Name() + f.Close() + os.Remove(name) + return nil +} + +// replaceExecutable writes bin next to target and swaps it in. On unix a +// rename over the running binary is atomic and safe. Windows cannot overwrite +// a running .exe but can rename it, so the old one is moved to .old +// (removed by CleanupOld on the next launch) before the new one is moved in. +func replaceExecutable(target string, bin []byte, windows bool) error { + dir := filepath.Dir(target) + tmp, err := os.CreateTemp(dir, ".orchestra-update-*") + if err != nil { + if errors.Is(err, fs.ErrPermission) { + return notWritable(dir, err) + } + return err + } + tmpName := tmp.Name() + ok := false + defer func() { + if !ok { + os.Remove(tmpName) + } + }() + if _, err := tmp.Write(bin); err != nil { + tmp.Close() + return fmt.Errorf("write update: %w", err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("write update: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("write update: %w", err) + } + if err := os.Chmod(tmpName, 0o755); err != nil { + return fmt.Errorf("chmod update: %w", err) + } + + if !windows { + if err := os.Rename(tmpName, target); err != nil { + return fmt.Errorf("replace %s: %w", target, err) + } + ok = true + return nil + } + + old := target + ".old" + _ = os.Remove(old) // stale leftover from a previous update + if err := os.Rename(target, old); err != nil { + return fmt.Errorf("move current executable aside: %w", err) + } + if err := os.Rename(tmpName, target); err != nil { + _ = os.Rename(old, target) // roll back + return fmt.Errorf("replace %s: %w", target, err) + } + ok = true + return nil +} + +// CleanupOld best-effort removes the .old left behind by a Windows +// update. Safe to call on every launch and on every platform. +func CleanupOld(exePath string) { + if exePath == "" { + return + } + if target, err := filepath.EvalSymlinks(exePath); err == nil { + exePath = target + } + _ = os.Remove(exePath + ".old") +} diff --git a/internal/update/update_test.go b/internal/update/update_test.go new file mode 100644 index 0000000..c51473b --- /dev/null +++ b/internal/update/update_test.go @@ -0,0 +1,503 @@ +package update + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestCompareVersions(t *testing.T) { + cases := []struct { + a, b string + want int + }{ + {"1.0.0", "1.0.0", 0}, + {"v1.0.0", "1.0.0", 0}, + {"1.0.1", "1.0.0", 1}, + {"1.1.0", "1.0.9", 1}, + {"2.0.0", "1.99.99", 1}, + {"0.10.0", "0.9.0", 1}, + {"0.9.0", "0.10.0", -1}, + {"1.0.0", "1.0.0-rc1", 1}, + {"1.0.0-rc1", "1.0.0", -1}, + {"1.0.0-rc2", "1.0.0-rc1", 1}, + {"1.0.0-rc.10", "1.0.0-rc.2", 1}, + {"1.0.0-alpha", "1.0.0-alpha.1", -1}, + {"1.0.0-1", "1.0.0-alpha", -1}, + {"1.0", "1.0.0", 0}, + } + for _, c := range cases { + got, err := Compare(c.a, c.b) + if err != nil { + t.Fatalf("Compare(%q,%q): %v", c.a, c.b, err) + } + if got != c.want { + t.Errorf("Compare(%q,%q) = %d, want %d", c.a, c.b, got, c.want) + } + } + for _, bad := range []string{"", "dev", "x.y.z", "1", "1.2.3.4", "1.-2.3"} { + if _, err := Compare(bad, "1.0.0"); err == nil { + t.Errorf("Compare(%q) should fail", bad) + } + } +} + +func TestCheckable(t *testing.T) { + for v, want := range map[string]bool{ + "0.8.0": true, + "v0.8.0": true, + "0.9.0-rc1": true, + "dev": false, + "": false, + "garbage": false, + "0.8.1-next": false, + "0.8.1-next.3": false, + } { + if got := Checkable(v); got != want { + t.Errorf("Checkable(%q) = %v, want %v", v, got, want) + } + } +} + +// releaseServer serves /repos//releases/latest with the given JSON body. +func releaseServer(t *testing.T, status int, body string, hits *int32) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits != nil { + atomic.AddInt32(hits, 1) + } + if r.URL.Path != "/repos/"+DefaultRepo+"/releases/latest" { + http.NotFound(w, r) + return + } + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestCheck(t *testing.T) { + body := `{"tag_name":"v0.9.0","prerelease":false,"assets":[{"name":"checksums.txt","browser_download_url":"http://x/checksums.txt","size":10}]}` + srv := releaseServer(t, 200, body, nil) + c := &Client{APIBase: srv.URL} + + rel, newer, err := c.Check(context.Background(), "0.8.0") + if err != nil { + t.Fatal(err) + } + if !newer || rel.Version != "0.9.0" || rel.Tag != "v0.9.0" || len(rel.Assets) != 1 || rel.Assets[0].URL != "http://x/checksums.txt" { + t.Fatalf("unexpected: newer=%v rel=%+v", newer, rel) + } + for _, cur := range []string{"0.9.0", "v0.9.0", "1.0.0"} { + if _, newer, err := c.Check(context.Background(), cur); err != nil || newer { + t.Errorf("current %s: newer=%v err=%v, want false/nil", cur, newer, err) + } + } + // A prerelease of the same version is upgraded to the stable. + if _, newer, _ := c.Check(context.Background(), "0.9.0-rc1"); !newer { + t.Error("0.9.0-rc1 should be offered 0.9.0") + } +} + +func TestCheckPrereleaseNeverOfferedOverStable(t *testing.T) { + for _, body := range []string{ + `{"tag_name":"v1.0.0-rc1","prerelease":true,"assets":[]}`, + `{"tag_name":"v1.0.0-rc1","prerelease":false,"assets":[]}`, + `{"tag_name":"v1.0.0","prerelease":true,"assets":[]}`, + } { + c := &Client{APIBase: releaseServer(t, 200, body, nil).URL} + if _, newer, err := c.Check(context.Background(), "0.8.0"); err != nil || newer { + t.Errorf("%s: newer=%v err=%v, want false/nil", body, newer, err) + } + } + // ...but a prerelease build may move to a newer prerelease. + c := &Client{APIBase: releaseServer(t, 200, `{"tag_name":"v1.0.0-rc2","prerelease":true}`, nil).URL} + if _, newer, _ := c.Check(context.Background(), "1.0.0-rc1"); !newer { + t.Error("1.0.0-rc1 should be offered 1.0.0-rc2") + } +} + +func TestCheckSkipsUncheckableWithoutNetwork(t *testing.T) { + var hits int32 + c := &Client{APIBase: releaseServer(t, 200, `{"tag_name":"v9.9.9"}`, &hits).URL} + for _, cur := range []string{"dev", "", "0.8.1-next", "nonsense"} { + if _, newer, err := c.Check(context.Background(), cur); err != nil || newer { + t.Errorf("%q: newer=%v err=%v", cur, newer, err) + } + } + if hits != 0 { + t.Fatalf("server was hit %d times for uncheckable versions", hits) + } +} + +func TestCheckErrors(t *testing.T) { + c := &Client{APIBase: releaseServer(t, 403, `rate limited`, nil).URL} + if _, _, err := c.Check(context.Background(), "0.8.0"); err == nil { + t.Error("non-200 should error") + } + c = &Client{APIBase: releaseServer(t, 200, `not json`, nil).URL} + if _, _, err := c.Check(context.Background(), "0.8.0"); err == nil { + t.Error("bad JSON should error") + } + c = &Client{APIBase: releaseServer(t, 200, `{"tag_name":"latest"}`, nil).URL} + if _, _, err := c.Check(context.Background(), "0.8.0"); err == nil { + t.Error("unparseable tag should error") + } +} + +func TestStateThrottle(t *testing.T) { + path := filepath.Join(t.TempDir(), "sub", "update.json") + st := LoadState(path) // missing → zero + now := time.Date(2026, 9, 26, 12, 0, 0, 0, time.UTC) + if !st.Due(now) { + t.Fatal("zero state should be due") + } + st.LastCheck = now + st.Latest = "0.9.0" + if err := SaveState(path, st); err != nil { + t.Fatal(err) + } + got := LoadState(path) + if !got.LastCheck.Equal(now) || got.Latest != "0.9.0" { + t.Fatalf("round trip: %+v", got) + } + if got.Due(now.Add(23 * time.Hour)) { + t.Error("should not be due after 23h") + } + if !got.Due(now.Add(24 * time.Hour)) { + t.Error("should be due after 24h") + } + if got.Due(now.Add(-time.Hour)) { + t.Error("clock skew backwards should not trigger a check") + } + if err := os.WriteFile(path, []byte("{corrupt"), 0o644); err != nil { + t.Fatal(err) + } + if !LoadState(path).Due(now) { + t.Error("corrupt state should be treated as due") + } +} + +// ---- Install ---- + +type entry struct { + name string + body string + dir bool +} + +func makeTarGz(t *testing.T, entries []entry) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, e := range entries { + hdr := &tar.Header{Name: e.name, Mode: 0o755, Size: int64(len(e.body)), Typeflag: tar.TypeReg} + if e.dir { + hdr = &tar.Header{Name: e.name, Mode: 0o755, Typeflag: tar.TypeDir} + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if !e.dir { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func makeZip(t *testing.T, entries []entry) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, e := range entries { + w, err := zw.Create(e.name) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func sha(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// fixture serves an archive and a checksums file and returns a matching Release. +func fixture(t *testing.T, version, goos, goarch string, archive []byte, checksums string) Release { + t.Helper() + ext := "tar.gz" + if goos == "windows" { + ext = "zip" + } + name := fmt.Sprintf("orchestra_%s_%s_%s.%s", version, goos, goarch, ext) + if checksums == "" { + checksums = fmt.Sprintf("%s %s\n%s orchestra_%s_other_os.tar.gz\n", sha(archive), name, strings.Repeat("0", 64), version) + } + mux := http.NewServeMux() + mux.HandleFunc("/dl/"+name, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(archive) }) + mux.HandleFunc("/dl/checksums.txt", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(checksums)) }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return Release{ + Tag: "v" + version, + Version: version, + Assets: []Asset{ + {Name: name, URL: srv.URL + "/dl/" + name, Size: int64(len(archive))}, + {Name: "checksums.txt", URL: srv.URL + "/dl/checksums.txt"}, + }, + } +} + +func fakeExe(t *testing.T, name string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte("OLD"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func assertFile(t *testing.T, path, want string) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(b) != want { + t.Fatalf("%s = %q, want %q", path, b, want) + } +} + +func TestInstallTarGz(t *testing.T) { + arc := makeTarGz(t, []entry{ + {name: "LICENSE", body: "mit"}, + {name: "README.md", body: "readme"}, + {name: "orchestra", body: "NEW-BINARY"}, + }) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + exe := fakeExe(t, "orchestra") + c := &Client{GOOS: "linux", GOARCH: "amd64"} + if err := c.Install(context.Background(), rel, exe); err != nil { + t.Fatal(err) + } + assertFile(t, exe, "NEW-BINARY") + if runtime.GOOS != "windows" { + fi, _ := os.Stat(exe) + if fi.Mode().Perm() != 0o755 { + t.Errorf("mode = %v, want 0755", fi.Mode().Perm()) + } + } + // No leftover temp files next to the exe. + ents, _ := os.ReadDir(filepath.Dir(exe)) + if len(ents) != 1 { + t.Errorf("leftover files: %v", ents) + } +} + +func TestInstallTarGzNestedDir(t *testing.T) { + arc := makeTarGz(t, []entry{ + {name: "orchestra_0.9.0/", dir: true}, + {name: "orchestra_0.9.0/orchestra", body: "NESTED"}, + }) + rel := fixture(t, "0.9.0", "darwin", "arm64", arc, "") + exe := fakeExe(t, "orchestra") + c := &Client{GOOS: "darwin", GOARCH: "arm64"} + if err := c.Install(context.Background(), rel, exe); err != nil { + t.Fatal(err) + } + assertFile(t, exe, "NESTED") +} + +func TestInstallFollowsSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks need privileges on windows") + } + arc := makeTarGz(t, []entry{{name: "orchestra", body: "NEW"}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + real := fakeExe(t, "orchestra") + link := filepath.Join(t.TempDir(), "orchestra-link") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + c := &Client{GOOS: "linux", GOARCH: "amd64"} + if err := c.Install(context.Background(), rel, link); err != nil { + t.Fatal(err) + } + assertFile(t, real, "NEW") + if fi, err := os.Lstat(link); err != nil || fi.Mode()&os.ModeSymlink == 0 { + t.Error("symlink should be left in place") + } +} + +func TestInstallZipWindowsStrategy(t *testing.T) { + arc := makeZip(t, []entry{ + {name: "LICENSE", body: "mit"}, + {name: "orchestra.exe", body: "NEW-EXE"}, + }) + rel := fixture(t, "0.9.0", "windows", "amd64", arc, "") + exe := fakeExe(t, "orchestra.exe") + // A stale .old from a previous update must not block the rename. + if err := os.WriteFile(exe+".old", []byte("STALE"), 0o644); err != nil { + t.Fatal(err) + } + c := &Client{GOOS: "windows", GOARCH: "amd64"} + if err := c.Install(context.Background(), rel, exe); err != nil { + t.Fatal(err) + } + assertFile(t, exe, "NEW-EXE") + assertFile(t, exe+".old", "OLD") + + CleanupOld(exe) + if _, err := os.Stat(exe + ".old"); !os.IsNotExist(err) { + t.Errorf(".old should be removed, stat err = %v", err) + } + CleanupOld(exe) // idempotent +} + +func TestInstallChecksumMismatch(t *testing.T) { + arc := makeTarGz(t, []entry{{name: "orchestra", body: "EVIL"}}) + name := "orchestra_0.9.0_linux_amd64.tar.gz" + rel := fixture(t, "0.9.0", "linux", "amd64", arc, strings.Repeat("a", 64)+" "+name+"\n") + exe := fakeExe(t, "orchestra") + err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe) + if err == nil || !strings.Contains(err.Error(), "checksum") { + t.Fatalf("want checksum error, got %v", err) + } + assertFile(t, exe, "OLD") +} + +func TestInstallMissingChecksumEntry(t *testing.T) { + arc := makeTarGz(t, []entry{{name: "orchestra", body: "NEW"}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, strings.Repeat("a", 64)+" something_else.tar.gz\n") + exe := fakeExe(t, "orchestra") + err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe) + if err == nil || !strings.Contains(err.Error(), "checksum") { + t.Fatalf("want missing checksum error, got %v", err) + } + assertFile(t, exe, "OLD") +} + +func TestInstallMissingChecksumsAsset(t *testing.T) { + arc := makeTarGz(t, []entry{{name: "orchestra", body: "NEW"}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + rel.Assets = rel.Assets[:1] + exe := fakeExe(t, "orchestra") + if err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe); err == nil { + t.Fatal("want error when checksums.txt is missing") + } + assertFile(t, exe, "OLD") +} + +func TestInstallMissingPlatformAsset(t *testing.T) { + arc := makeZip(t, []entry{{name: "orchestra.exe", body: "NEW"}}) + rel := fixture(t, "0.9.0", "windows", "amd64", arc, "") + exe := fakeExe(t, "orchestra.exe") + err := (&Client{GOOS: "windows", GOARCH: "arm64"}).Install(context.Background(), rel, exe) + if err == nil || !strings.Contains(err.Error(), "windows/arm64") { + t.Fatalf("want missing-asset error naming the platform, got %v", err) + } +} + +func TestInstallRejectsTraversal(t *testing.T) { + for name, arc := range map[string][]byte{ + "tar dotdot": makeTarGz(t, []entry{{name: "../orchestra", body: "EVIL"}, {name: "orchestra", body: "NEW"}}), + "tar absolute": makeTarGz(t, []entry{{name: "/tmp/orchestra", body: "EVIL"}}), + } { + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + exe := fakeExe(t, "orchestra") + err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe) + if err == nil || !strings.Contains(err.Error(), "unsafe") { + t.Errorf("%s: want unsafe-path error, got %v", name, err) + } + assertFile(t, exe, "OLD") + } + zarc := makeZip(t, []entry{{name: "..\\..\\orchestra.exe", body: "EVIL"}}) + rel := fixture(t, "0.9.0", "windows", "amd64", zarc, "") + exe := fakeExe(t, "orchestra.exe") + if err := (&Client{GOOS: "windows", GOARCH: "amd64"}).Install(context.Background(), rel, exe); err == nil || !strings.Contains(err.Error(), "unsafe") { + t.Errorf("zip traversal: want unsafe-path error, got %v", err) + } +} + +func TestInstallNoBinaryInArchive(t *testing.T) { + arc := makeTarGz(t, []entry{{name: "README.md", body: "x"}, {name: "deep/er/orchestra", body: "too deep"}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + exe := fakeExe(t, "orchestra") + if err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe); err == nil { + t.Fatal("want error when the archive has no orchestra binary") + } + assertFile(t, exe, "OLD") +} + +func TestInstallUnwritableDir(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("permission bits not enforced here") + } + arc := makeTarGz(t, []entry{{name: "orchestra", body: "NEW"}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + exe := fakeExe(t, "orchestra") + dir := filepath.Dir(exe) + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + err := (&Client{GOOS: "linux", GOARCH: "amd64"}).Install(context.Background(), rel, exe) + if !errors.Is(err, ErrNotWritable) || !strings.Contains(err.Error(), "reinstall") { + t.Fatalf("want ErrNotWritable with a hint, got %v", err) + } + assertFile(t, exe, "OLD") +} + +func TestInstallSizeLimit(t *testing.T) { + arc := makeTarGz(t, []entry{{name: "orchestra", body: strings.Repeat("x", 4096)}}) + rel := fixture(t, "0.9.0", "linux", "amd64", arc, "") + exe := fakeExe(t, "orchestra") + c := &Client{GOOS: "linux", GOARCH: "amd64", MaxDownload: 64} + if err := c.Install(context.Background(), rel, exe); err == nil || !strings.Contains(err.Error(), "limit") { + t.Fatalf("want size-limit error, got %v", err) + } +} + +// Guard: the JSON field names match GitHub's API. +func TestReleaseJSONShape(t *testing.T) { + var r ghRelease + if err := json.Unmarshal([]byte(`{"tag_name":"v1.2.3","prerelease":true,"assets":[{"name":"a","browser_download_url":"u","size":3}]}`), &r); err != nil { + t.Fatal(err) + } + if r.TagName != "v1.2.3" || !r.Prerelease || r.Assets[0].URL != "u" || r.Assets[0].Size != 3 { + t.Fatalf("%+v", r) + } +} diff --git a/internal/update/version.go b/internal/update/version.go new file mode 100644 index 0000000..f82bd15 --- /dev/null +++ b/internal/update/version.go @@ -0,0 +1,138 @@ +package update + +import ( + "fmt" + "strconv" + "strings" +) + +// semver is a parsed MAJOR.MINOR.PATCH[-PRERELEASE] version. Build metadata +// (+...) is ignored, as semver requires. +type semver struct { + nums [3]int + pre []string +} + +// parseVersion accepts an optional leading "v" and a missing patch ("1.2"). +func parseVersion(s string) (semver, error) { + var v semver + raw := s + s = strings.TrimPrefix(strings.TrimSpace(s), "v") + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + if i := strings.IndexByte(s, '-'); i >= 0 { + if s[i+1:] == "" { + return v, fmt.Errorf("invalid version %q", raw) + } + v.pre = strings.Split(s[i+1:], ".") + s = s[:i] + } + parts := strings.Split(s, ".") + if len(parts) < 2 || len(parts) > 3 { + return v, fmt.Errorf("invalid version %q", raw) + } + for i, p := range parts { + n, err := strconv.Atoi(p) + if err != nil || n < 0 || p == "" || p[0] == '+' { + return v, fmt.Errorf("invalid version %q", raw) + } + v.nums[i] = n + } + return v, nil +} + +func (v semver) prerelease() bool { return len(v.pre) > 0 } + +// snapshot reports a GoReleaser snapshot build ("-next"). +func (v semver) snapshot() bool { + return len(v.pre) > 0 && v.pre[0] == "next" +} + +func (v semver) compare(o semver) int { + for i := 0; i < 3; i++ { + if v.nums[i] != o.nums[i] { + return sign(v.nums[i] - o.nums[i]) + } + } + // A release ranks above any prerelease of the same version. + switch { + case !v.prerelease() && !o.prerelease(): + return 0 + case !v.prerelease(): + return 1 + case !o.prerelease(): + return -1 + } + for i := 0; i < len(v.pre) && i < len(o.pre); i++ { + if c := comparePreIdent(v.pre[i], o.pre[i]); c != 0 { + return c + } + } + return sign(len(v.pre) - len(o.pre)) +} + +// comparePreIdent follows semver §11 for well-formed identifiers: numeric ones +// compare numerically and rank below alphanumeric ones. Alphanumeric +// identifiers with a trailing number ("rc2" vs "rc10") compare that number +// numerically, which matches how tags like v1.0.0-rc10 are usually meant. +func comparePreIdent(a, b string) int { + na, ea := strconv.Atoi(a) + nb, eb := strconv.Atoi(b) + switch { + case ea == nil && eb == nil: + return sign(na - nb) + case ea == nil: + return -1 + case eb == nil: + return 1 + } + pa, sa := splitTrailingNum(a) + pb, sb := splitTrailingNum(b) + if pa == pb && sa != "" && sb != "" { + x, _ := strconv.Atoi(sa) + y, _ := strconv.Atoi(sb) + return sign(x - y) + } + return strings.Compare(a, b) +} + +func splitTrailingNum(s string) (prefix, num string) { + i := len(s) + for i > 0 && s[i-1] >= '0' && s[i-1] <= '9' { + i-- + } + return s[:i], s[i:] +} + +func sign(n int) int { + switch { + case n > 0: + return 1 + case n < 0: + return -1 + } + return 0 +} + +// Compare returns -1, 0 or 1 as version a is lower than, equal to or higher +// than b. Both may carry a leading "v". +func Compare(a, b string) (int, error) { + va, err := parseVersion(a) + if err != nil { + return 0, err + } + vb, err := parseVersion(b) + if err != nil { + return 0, err + } + return va.compare(vb), nil +} + +// Checkable reports whether a build of this version should look for updates. +// Development builds ("dev", unparseable) and GoReleaser snapshots ("-next") +// never do: there is no meaningful release to compare them with. +func Checkable(current string) bool { + v, err := parseVersion(current) + return err == nil && !v.snapshot() +} From baee2999c32353ffa80cd209e710a463ff82b69d Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:01:21 +0000 Subject: [PATCH 11/25] cli: offer updates on launch and add `orchestra update` A root PersistentPreRunE checks for a newer release (2s timeout, throttled to once a day) only in interactive terminals, never in CI, for dev builds, with ORCHESTRA_NO_UPDATE_CHECK=1, or for help, completion, --version and update itself. Accepting installs, prints 'Updated to vX. Please relaunch orchestra.' and exits 0; declining or a failed install continues with the requested command. `orchestra update` forces a check (--check to report, -y to skip the prompt). Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/root.go | 8 + cmd/orchestra/update.go | 274 ++++++++++++++++++++++++++++++ cmd/orchestra/update_test.go | 311 +++++++++++++++++++++++++++++++++++ 3 files changed, 593 insertions(+) create mode 100644 cmd/orchestra/update.go create mode 100644 cmd/orchestra/update_test.go diff --git a/cmd/orchestra/root.go b/cmd/orchestra/root.go index 9cfcace..947e385 100644 --- a/cmd/orchestra/root.go +++ b/cmd/orchestra/root.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "github.com/crossben/orchestra-code/internal/config" "github.com/crossben/orchestra-code/internal/gitutil" @@ -29,6 +30,12 @@ func newRootCmd() *cobra.Command { Version: version, SilenceUsage: true, SilenceErrors: true, + // Before any command: offer a newer release when running interactively. + // Never fails — update problems must not block the requested command. + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + launchCheck(cmd, realUpdateEnv(), newUpdater(), os.Exit) + return nil + }, // No subcommand → start the interactive shell. RunE: func(cmd *cobra.Command, args []string) error { return runShell(cmd) @@ -45,6 +52,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newHistoryCmd()) root.AddCommand(newBenchmarkCmd()) root.AddCommand(newDashboardCmd()) + root.AddCommand(newUpdateCmd()) return root } diff --git a/cmd/orchestra/update.go b/cmd/orchestra/update.go new file mode 100644 index 0000000..3a6ccb7 --- /dev/null +++ b/cmd/orchestra/update.go @@ -0,0 +1,274 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/crossben/orchestra-code/internal/ui" + "github.com/crossben/orchestra-code/internal/update" + "github.com/spf13/cobra" +) + +// autoCheckTimeout bounds the launch-time release check so a slow network +// never delays startup noticeably. +const autoCheckTimeout = 2 * time.Second + +// installTimeout bounds the download + replace once the user said yes. +const installTimeout = 5 * time.Minute + +// updateEnv is everything the launch hook's skip decision depends on, so it +// can be tested without a terminal. +type updateEnv struct { + stdinTTY bool + stdoutTTY bool + getenv func(string) string +} + +func realUpdateEnv() updateEnv { + return updateEnv{stdinTTY: isTTY(os.Stdin), stdoutTTY: isTTY(os.Stdout), getenv: os.Getenv} +} + +func isTTY(f *os.File) bool { + fi, err := f.Stat() + return err == nil && fi.Mode()&os.ModeCharDevice != 0 +} + +// skipUpdateCommands never trigger the launch check. +var skipUpdateCommands = map[string]bool{ + "help": true, + "completion": true, + "update": true, + cobra.ShellCompRequestCmd: true, // __complete + cobra.ShellCompNoDescRequestCmd: true, // __completeNoDesc +} + +// shouldAutoCheck decides whether the launch hook may touch the network. +// Anything non-interactive (pipes, scripts, CI, tests) is skipped. +func shouldAutoCheck(cmd *cobra.Command, current string, env updateEnv) bool { + if !env.stdinTTY || !env.stdoutTTY { + return false + } + if env.getenv("CI") != "" { + return false + } + if v := env.getenv("ORCHESTRA_NO_UPDATE_CHECK"); v != "" && v != "0" && !strings.EqualFold(v, "false") { + return false + } + if !update.Checkable(current) { + return false + } + for c := cmd; c != nil; c = c.Parent() { + if skipUpdateCommands[c.Name()] { + return false + } + } + for _, f := range []string{"version", "help"} { + if fl := cmd.Flags().Lookup(f); fl != nil && fl.Changed { + return false + } + } + return true +} + +// updater runs the check → prompt → install flow for both the launch hook and +// `orchestra update`. All I/O is injectable for tests. +type updater struct { + client *update.Client + statePath string // "" disables throttle persistence + current string + exePath string + in io.Reader + out io.Writer + now func() time.Time +} + +func newUpdater() *updater { + exe, _ := os.Executable() + state, _ := update.DefaultStatePath() + return &updater{ + client: &update.Client{UserAgent: "orchestra/" + version}, + statePath: state, + current: version, + exePath: exe, + in: os.Stdin, + out: os.Stdout, + now: time.Now, + } +} + +// outcome of an update offer. +type updateOutcome int + +const ( + outcomeDeclined updateOutcome = iota // newer release, user said no + outcomeInstalled // replaced the binary — caller must exit + outcomeFailed // newer release, install failed +) + +// check performs the (optionally throttled) release check and records the +// attempt. It returns the release and whether it is newer. +func (u *updater) check(ctx context.Context, throttled bool) (update.Release, bool, error) { + var st update.State + if u.statePath != "" { + st = update.LoadState(u.statePath) + } + if throttled && !st.Due(u.now()) { + return update.Release{}, false, nil + } + // Record the attempt before the network call: an offline or slow network + // then costs at most one short wait per interval. + st.LastCheck = u.now() + rel, newer, err := u.client.Check(ctx, u.current) + if err == nil { + st.Latest = rel.Version + } + if u.statePath != "" { + _ = update.SaveState(u.statePath, st) + } + return rel, newer, err +} + +// offer prompts (unless assumeYes) and installs rel. +func (u *updater) offer(ctx context.Context, rel update.Release, assumeYes bool) updateOutcome { + if !assumeYes { + fmt.Fprintf(u.out, "%s Orchestra v%s is available (you have v%s). Download and install now? [Y/n] ", + ui.Accent("↑"), rel.Version, strings.TrimPrefix(u.current, "v")) + if !confirmDefaultYes(readLine(u.in)) { + fmt.Fprintln(u.out, ui.Dim(" Skipped — you'll be reminded again in a day. Run `orchestra update` any time.")) + return outcomeDeclined + } + } + sp := ui.Spin(fmt.Sprintf("Downloading orchestra v%s…", rel.Version)) + err := u.client.Install(ctx, rel, u.exePath) + sp.Stop() + if err != nil { + fmt.Fprintf(u.out, "%s update failed: %v\n", ui.Danger("✗"), err) + return outcomeFailed + } + fmt.Fprintf(u.out, "%s Updated to v%s. Please relaunch orchestra.\n", ui.Success("✓"), rel.Version) + return outcomeInstalled +} + +// launchCheck is the root PersistentPreRunE body. It never returns an error: +// update problems must not block the command the user asked for. +func launchCheck(cmd *cobra.Command, env updateEnv, u *updater, exit func(int)) { + update.CleanupOld(u.exePath) + if !shouldAutoCheck(cmd, u.current, env) { + return + } + parent := cmd.Context() + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithTimeout(parent, autoCheckTimeout) + rel, newer, err := u.check(ctx, true) + cancel() + if err != nil { + if env.getenv("ORCHESTRA_DEBUG") != "" { + fmt.Fprintf(os.Stderr, "orchestra: update check failed: %v\n", err) + } + return + } + if !newer { + return + } + ictx, icancel := context.WithTimeout(parent, installTimeout) + defer icancel() + if u.offer(ictx, rel, false) == outcomeInstalled { + // Do not keep running the old binary. + exit(0) + } +} + +func newUpdateCmd() *cobra.Command { + var checkOnly, yes bool + cmd := &cobra.Command{ + Use: "update", + Short: "Check for a newer Orchestra release and install it", + Long: "Checks GitHub Releases for a newer Orchestra, and (after confirmation) downloads it,\n" + + "verifies its SHA-256 against checksums.txt and replaces this executable.\n" + + "Orchestra also checks automatically at most once a day in interactive terminals;\n" + + "set ORCHESTRA_NO_UPDATE_CHECK=1 to disable that.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runUpdate(cmd.Context(), newUpdater(), checkOnly, yes) + }, + } + cmd.Flags().BoolVar(&checkOnly, "check", false, "only report whether an update is available") + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "install without asking") + return cmd +} + +// runUpdate is `orchestra update`: a forced (unthrottled) check. +func runUpdate(ctx context.Context, u *updater, checkOnly, yes bool) error { + if ctx == nil { + ctx = context.Background() + } + if !update.Checkable(u.current) { + fmt.Fprintf(u.out, "This is a development build (%s); updates are only offered for release builds.\n", u.current) + return nil + } + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + sp := ui.Spin("Checking for updates…") + rel, newer, err := u.check(cctx, false) + sp.Stop() + cancel() + if err != nil { + return fmt.Errorf("update check failed: %w", err) + } + if !newer { + fmt.Fprintf(u.out, "Orchestra v%s is up to date.\n", strings.TrimPrefix(u.current, "v")) + return nil + } + if checkOnly { + fmt.Fprintf(u.out, "Orchestra v%s is available (you have v%s). Run `orchestra update` to install it.\n", + rel.Version, strings.TrimPrefix(u.current, "v")) + return nil + } + ictx, icancel := context.WithTimeout(ctx, installTimeout) + defer icancel() + switch u.offer(ictx, rel, yes) { + case outcomeFailed: + return errors.New("update not installed") + } + // Installed: returning ends the process (exit 0), so the old binary does + // not keep running. Declined: nothing to do. + return nil +} + +// readLine reads up to a newline one byte at a time, so no input meant for +// whatever runs next (e.g. the shell) is buffered away. +func readLine(r io.Reader) string { + var b strings.Builder + buf := make([]byte, 1) + for b.Len() < 256 { + n, err := r.Read(buf) + if n == 1 { + if buf[0] == '\n' { + break + } + b.WriteByte(buf[0]) + } + if err != nil { + if b.Len() == 0 { + return "n" // EOF with no answer: treat as no + } + break + } + } + return strings.TrimSpace(b.String()) +} + +// confirmDefaultYes: Enter, y, yes → true. +func confirmDefaultYes(ans string) bool { + switch strings.ToLower(strings.TrimSpace(ans)) { + case "", "y", "yes": + return true + } + return false +} diff --git a/cmd/orchestra/update_test.go b/cmd/orchestra/update_test.go new file mode 100644 index 0000000..90ec27c --- /dev/null +++ b/cmd/orchestra/update_test.go @@ -0,0 +1,311 @@ +package main + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/crossben/orchestra-code/internal/update" + "github.com/spf13/cobra" +) + +func ttyEnv(vars map[string]string) updateEnv { + return updateEnv{stdinTTY: true, stdoutTTY: true, getenv: func(k string) string { return vars[k] }} +} + +// findCmd resolves a command path on a fresh root, like cobra would. +func findCmd(t *testing.T, args ...string) *cobra.Command { + t.Helper() + root := newRootCmd() + root.InitDefaultHelpCmd() + root.InitDefaultCompletionCmd() + cmd, _, err := root.Find(args) + if err != nil { + t.Fatalf("find %v: %v", args, err) + } + return cmd +} + +func TestShouldAutoCheck(t *testing.T) { + run := findCmd(t, "run") + cases := []struct { + name string + cmd *cobra.Command + version string + env updateEnv + want bool + }{ + {"interactive release build", run, "0.8.0", ttyEnv(nil), true}, + {"root command", findCmd(t), "0.8.0", ttyEnv(nil), true}, + {"stdin piped", run, "0.8.0", updateEnv{stdinTTY: false, stdoutTTY: true, getenv: ttyEnv(nil).getenv}, false}, + {"stdout piped", run, "0.8.0", updateEnv{stdinTTY: true, stdoutTTY: false, getenv: ttyEnv(nil).getenv}, false}, + {"CI", run, "0.8.0", ttyEnv(map[string]string{"CI": "true"}), false}, + {"opt-out", run, "0.8.0", ttyEnv(map[string]string{"ORCHESTRA_NO_UPDATE_CHECK": "1"}), false}, + {"opt-out explicitly off", run, "0.8.0", ttyEnv(map[string]string{"ORCHESTRA_NO_UPDATE_CHECK": "0"}), true}, + {"dev build", run, "dev", ttyEnv(nil), false}, + {"snapshot build", run, "0.8.1-next", ttyEnv(nil), false}, + {"unparseable", run, "abc123", ttyEnv(nil), false}, + {"update command", findCmd(t, "update"), "0.8.0", ttyEnv(nil), false}, + {"help command", findCmd(t, "help"), "0.8.0", ttyEnv(nil), false}, + {"completion", findCmd(t, "completion", "bash"), "0.8.0", ttyEnv(nil), false}, + } + for _, c := range cases { + if got := shouldAutoCheck(c.cmd, c.version, c.env); got != c.want { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + } + } + + // --version / --help flags. + root := newRootCmd() + root.InitDefaultVersionFlag() + _ = root.ParseFlags([]string{"--version"}) + if shouldAutoCheck(root, "0.8.0", ttyEnv(nil)) { + t.Error("--version should skip the check") + } +} + +func TestConfirmAndReadLine(t *testing.T) { + for in, want := range map[string]bool{"\n": true, "y\n": true, "YES\n": true, " y \r\n": true, "n\n": false, "no\n": false, "": false, "nope": false} { + if got := confirmDefaultYes(readLine(strings.NewReader(in))); got != want { + t.Errorf("answer %q: got %v, want %v", in, got, want) + } + } + // readLine must not consume past the newline. + r := strings.NewReader("y\nnext command\n") + readLine(r) + if rest := readLine(r); rest != "next command" { + t.Errorf("remaining input = %q", rest) + } +} + +// fakeReleases serves a GitHub-like API plus the release assets. +type fakeReleases struct { + srv *httptest.Server + apiHits int32 + fail atomic.Bool + latestV string + archive []byte + checksum string +} + +func newFakeReleases(t *testing.T, latest string) *fakeReleases { + t.Helper() + f := &fakeReleases{latestV: latest} + binName := "orchestra" + if runtime.GOOS == "windows" { + binName = "orchestra.exe" + } + name := update.AssetName(latest, runtime.GOOS, runtime.GOARCH) + if strings.HasSuffix(name, ".zip") { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create(binName) + _, _ = w.Write([]byte("NEW-" + latest)) + _ = zw.Close() + f.archive = buf.Bytes() + } else { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + body := []byte("NEW-" + latest) + _ = tw.WriteHeader(&tar.Header{Name: binName, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg}) + _, _ = tw.Write(body) + _ = tw.Close() + _ = gz.Close() + f.archive = buf.Bytes() + } + sum := sha256.Sum256(f.archive) + f.checksum = hex.EncodeToString(sum[:]) + " " + name + "\n" + + mux := http.NewServeMux() + mux.HandleFunc("/repos/"+update.DefaultRepo+"/releases/latest", func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&f.apiHits, 1) + if f.fail.Load() { + http.Error(w, "boom", http.StatusInternalServerError) + return + } + fmt.Fprintf(w, `{"tag_name":"v%s","assets":[{"name":%q,"browser_download_url":"%s/dl/a"},{"name":"checksums.txt","browser_download_url":"%s/dl/sums"}]}`, + f.latestV, name, f.srv.URL, f.srv.URL) + }) + mux.HandleFunc("/dl/a", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(f.archive) }) + mux.HandleFunc("/dl/sums", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(f.checksum)) }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func testUpdater(t *testing.T, f *fakeReleases, current, answer string) (*updater, *bytes.Buffer) { + t.Helper() + exe := filepath.Join(t.TempDir(), "orchestra") + if runtime.GOOS == "windows" { + exe += ".exe" + } + if err := os.WriteFile(exe, []byte("OLD"), 0o755); err != nil { + t.Fatal(err) + } + out := &bytes.Buffer{} + return &updater{ + client: &update.Client{APIBase: f.srv.URL}, + statePath: filepath.Join(t.TempDir(), "update.json"), + current: current, + exePath: exe, + in: strings.NewReader(answer), + out: out, + now: func() time.Time { return time.Date(2026, 9, 26, 12, 0, 0, 0, time.UTC) }, + }, out +} + +func exeContent(t *testing.T, u *updater) string { + t.Helper() + b, err := os.ReadFile(u.exePath) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestLaunchCheckInstallsAndExits(t *testing.T) { + f := newFakeReleases(t, "0.9.0") + u, out := testUpdater(t, f, "0.8.0", "\n") + exited := -1 + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, func(code int) { exited = code }) + if exited != 0 { + t.Fatalf("exit code = %d, want 0 (output: %s)", exited, out) + } + if got := exeContent(t, u); got != "NEW-0.9.0" { + t.Fatalf("exe = %q", got) + } + s := out.String() + if !strings.Contains(s, "Orchestra v0.9.0 is available (you have v0.8.0). Download and install now? [Y/n]") || + !strings.Contains(s, "Updated to v0.9.0. Please relaunch orchestra.") { + t.Fatalf("output:\n%s", s) + } +} + +func TestLaunchCheckDeclineThenThrottled(t *testing.T) { + f := newFakeReleases(t, "0.9.0") + u, out := testUpdater(t, f, "0.8.0", "n\n") + exit := func(int) { t.Fatal("must not exit after declining") } + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, exit) + if exeContent(t, u) != "OLD" || !strings.Contains(out.String(), "Skipped") { + t.Fatalf("declined flow wrong; output:\n%s", out) + } + st := update.LoadState(u.statePath) + if st.Latest != "0.9.0" || st.LastCheck.IsZero() { + t.Fatalf("state not recorded: %+v", st) + } + + // Same day: no network, no prompt. + out.Reset() + u.in = strings.NewReader("\n") + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, exit) + if atomic.LoadInt32(&f.apiHits) != 1 || out.Len() != 0 { + t.Fatalf("throttle failed: hits=%d output=%q", f.apiHits, out) + } + + // A day later it asks again. + u.now = func() time.Time { return time.Date(2026, 9, 27, 12, 0, 1, 0, time.UTC) } + u.in = strings.NewReader("no\n") + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, exit) + if atomic.LoadInt32(&f.apiHits) != 2 || !strings.Contains(out.String(), "is available") { + t.Fatalf("expected a new prompt after 24h: hits=%d output=%q", f.apiHits, out) + } +} + +func TestLaunchCheckSilentOnErrorAndSkips(t *testing.T) { + f := newFakeReleases(t, "0.9.0") + f.fail.Store(true) + u, out := testUpdater(t, f, "0.8.0", "\n") + exit := func(int) { t.Fatal("must not exit") } + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, exit) + if out.Len() != 0 || exeContent(t, u) != "OLD" { + t.Fatalf("network errors must be silent; output=%q", out) + } + + // Skip conditions never reach the network. + f.fail.Store(false) + hits := atomic.LoadInt32(&f.apiHits) + for _, env := range []updateEnv{ + {stdinTTY: false, stdoutTTY: true, getenv: ttyEnv(nil).getenv}, + ttyEnv(map[string]string{"CI": "1"}), + ttyEnv(map[string]string{"ORCHESTRA_NO_UPDATE_CHECK": "1"}), + } { + u2, out2 := testUpdater(t, f, "0.8.0", "\n") + launchCheck(findCmd(t, "run"), env, u2, exit) + if out2.Len() != 0 { + t.Errorf("unexpected output %q", out2) + } + } + u3, _ := testUpdater(t, f, "dev", "\n") + launchCheck(findCmd(t, "run"), ttyEnv(nil), u3, exit) + if atomic.LoadInt32(&f.apiHits) != hits { + t.Fatalf("skip conditions hit the network %d times", f.apiHits-hits) + } +} + +func TestLaunchCheckInstallFailureContinues(t *testing.T) { + f := newFakeReleases(t, "0.9.0") + f.checksum = strings.Repeat("0", 64) + " " + update.AssetName("0.9.0", runtime.GOOS, runtime.GOARCH) + "\n" + u, out := testUpdater(t, f, "0.8.0", "y\n") + launchCheck(findCmd(t, "run"), ttyEnv(nil), u, func(int) { t.Fatal("must not exit on failed install") }) + if exeContent(t, u) != "OLD" || !strings.Contains(out.String(), "update failed") { + t.Fatalf("output:\n%s", out) + } +} + +func TestRunUpdateCommand(t *testing.T) { + f := newFakeReleases(t, "0.9.0") + + // --check only reports, even though the throttle would say "not due". + u, out := testUpdater(t, f, "0.8.0", "") + _ = update.SaveState(u.statePath, update.State{LastCheck: u.now()}) + if err := runUpdate(context.Background(), u, true, false); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "v0.9.0 is available") || exeContent(t, u) != "OLD" || atomic.LoadInt32(&f.apiHits) != 1 { + t.Fatalf("--check: hits=%d output=%q", f.apiHits, out) + } + + // -y installs without a prompt. + u, out = testUpdater(t, f, "0.8.0", "") + if err := runUpdate(context.Background(), u, false, true); err != nil { + t.Fatal(err) + } + if exeContent(t, u) != "NEW-0.9.0" || strings.Contains(out.String(), "[Y/n]") { + t.Fatalf("-y: output=%q", out) + } + + // Up to date. + u, out = testUpdater(t, f, "0.9.0", "") + if err := runUpdate(context.Background(), u, false, false); err != nil || !strings.Contains(out.String(), "up to date") { + t.Fatalf("up to date: err=%v output=%q", err, out) + } + + // Dev build explains itself without touching the network. + hits := atomic.LoadInt32(&f.apiHits) + u, out = testUpdater(t, f, "dev", "") + if err := runUpdate(context.Background(), u, false, false); err != nil || !strings.Contains(out.String(), "development build") || atomic.LoadInt32(&f.apiHits) != hits { + t.Fatalf("dev: err=%v output=%q", err, out) + } + + // Check failure is an error for the explicit command. + f.fail.Store(true) + u, _ = testUpdater(t, f, "0.8.0", "") + if err := runUpdate(context.Background(), u, false, false); err == nil { + t.Fatal("want error when the check fails") + } +} From 910f00c915c84d3b078883a892c3f0c5b51a4dd3 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:01:21 +0000 Subject: [PATCH 12/25] docs: document updating and the updater's release-asset contract Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++++++ docs/RELEASING.md | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/README.md b/README.md index 093a9ab..a6688e2 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,18 @@ go build -o bin/orchestra ./cmd/orchestra ./bin/orchestra init # write a starter orchestra.yaml ``` +## Updating + +Release builds check GitHub for a newer version at most once a day when started in an interactive +terminal, and ask `Download and install now? [Y/n]`. On yes, Orchestra downloads the archive for your +platform, verifies its SHA-256 against the release's `checksums.txt`, replaces its own executable, and +exits — relaunch to use the new version. Run `orchestra update` to check on demand (`--check` to only +report, `-y` to skip the prompt). + +The check never runs in pipes, scripts or CI (`CI` set), for dev/snapshot builds, or when +`ORCHESTRA_NO_UPDATE_CHECK=1`. Installed via Homebrew/Scoop or into a directory you can't write to? +Update the same way you installed instead. + ## Configuration Orchestra works with no config (built-in defaults for the common agents). To customize, `orchestra init` @@ -263,6 +275,7 @@ A config file overrides defaults and adds agents; matching names replace the bui | `orchestra history` | recent runs + preferred agent | | `orchestra agents` | list agents; `--probe` live-tests each can actually run | | `orchestra init` | write a starter `orchestra.yaml` | +| `orchestra update` | install the latest release; `--check` only reports, `-y` skips the prompt | `run` flags: `--agent`, `--test`, `--retries`, `--timeout`, `--force`. `do`: `--agent`, `--yes`, `--parallel`, `--jobs`. Global: `--dir`. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index e6a63b1..533ed56 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -23,6 +23,14 @@ GoReleaser. In ~2 minutes the [Releases page](https://github.com/crossben/orches The version is stamped into the binary at build time via `-ldflags "-X main.version="`, so `orchestra --version` reports the tag. +**The self-updater depends on these names.** `orchestra update` (and the daily launch check) reads +`/releases/latest`, downloads `orchestra___.tar.gz` (`.zip` on windows) and refuses to +install unless `checksums.txt` has a matching ` ` line. The archive must contain the +`orchestra` (`orchestra.exe`) binary at its root. Changing `archives.name_template`, the archive formats, +the binary name or `checksum.name_template` in `.goreleaser.yaml` breaks updates for every installed +version (they look for the old names), so treat them as a stable contract; `internal/update` (`AssetName`) +mirrors them. + **Versioning:** semver, `vMAJOR.MINOR.PATCH`. Pre-release suffixes (e.g. `v0.8.0-rc1`) are published as GitHub pre-releases automatically (`prerelease: auto`). From 6afd4397b719c949fccefb9abcf6195f73cc2961 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:02:15 +0000 Subject: [PATCH 13/25] do --parallel, benchmark: run outside git repositories Both commands now pick their isolation backend with worktree.New: git worktrees inside a repository (unchanged), folder copies elsewhere. The 'needs a git repository' errors are gone; the clean-tree check still applies inside a repo. In a plain folder the stray-write guard snapshots the base before each wave and restores it with fsdiff. The benchmark body moves into runBenchmark so it can be tested with fake agents. Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/benchmark.go | 191 ++++++++++++++++++---------------- cmd/orchestra/do.go | 81 +++++++++++--- cmd/orchestra/gitfree_test.go | 166 +++++++++++++++++++++++++++++ 3 files changed, 332 insertions(+), 106 deletions(-) create mode 100644 cmd/orchestra/gitfree_test.go diff --git a/cmd/orchestra/benchmark.go b/cmd/orchestra/benchmark.go index 3f11e53..63c5d99 100644 --- a/cmd/orchestra/benchmark.go +++ b/cmd/orchestra/benchmark.go @@ -17,6 +17,7 @@ import ( "github.com/crossben/orchestra-code/internal/memory" "github.com/crossben/orchestra-code/internal/scheduler" "github.com/crossben/orchestra-code/internal/ui" + "github.com/crossben/orchestra-code/internal/validate" "github.com/crossben/orchestra-code/internal/worktree" "github.com/spf13/cobra" ) @@ -57,8 +58,8 @@ func newBenchmarkCmd() *cobra.Command { Use: `benchmark ""`, Short: "Run one task through every agent (isolated) and rank the results", Long: "Run the same task from the same starting point through each available agent, each in its own\n" + - "git worktree, then print a leaderboard (validation, speed, retries, diff size). Offers to keep\n" + - "the winner. With --compare, runs each agent twice (principles off vs on) and reports how much\n" + + "isolated tree (a git worktree, or a directory copy in a plain folder), then print a leaderboard\n" + + "(validation, speed, retries, diff size). Offers to keep the winner. With --compare, runs each agent twice (principles off vs on) and reports how much\n" + "smaller the lean-code principles make the diff.", Args: cobra.ArbitraryArgs, RunE: func(cmd *cobra.Command, args []string) error { @@ -70,13 +71,14 @@ func newBenchmarkCmd() *cobra.Command { if err != nil { return err } - if !gitutil.IsRepo(flagDir) { - return fmt.Errorf("benchmark needs a git repository — every agent runs in an isolated worktree so their changes never collide") - } - if clean, err := gitutil.IsClean(flagDir); err != nil { - return err - } else if !clean { - return errDirty() + // Every agent runs isolated (git worktree in a repository, directory + // copy elsewhere); only a repository must start clean. + if gitutil.IsRepo(flagDir) { + if clean, err := gitutil.IsClean(flagDir); err != nil { + return err + } else if !clean { + return errDirty() + } } reg := cfg.BuildRegistry() @@ -99,86 +101,7 @@ func newBenchmarkCmd() *cobra.Command { jobs = len(agents) } - stages := stagesFor(cfg) - retries := cfg.RetryLimit() - - mgr, err := worktree.NewManager(flagDir) - if err != nil { - return err - } - defer mgr.Cleanup() - - names := make([]string, len(agents)) - for i, a := range agents { - names[i] = a.Name() - } - fmt.Printf("%s benchmarking %q across %s\n", ui.Accent("▸"), task, ui.Agent(strings.Join(names, ", "))) - - // Run each agent in its own worktree, in parallel. - results := make([]benchResult, len(agents)) - scheduler.Bounded(cmd.Context(), jobs, len(agents), func(i int) error { - a := agents[i] - results[i].agent = a.Name() - tree, aerr := mgr.Add("bench-"+a.Name(), "HEAD") - if aerr != nil { - results[i].err = aerr - return aerr - } - results[i].tree = tree - results[i].created = true - - start := time.Now() - out, rerr := engine.ExecuteHeadless(cmd.Context(), engine.Options{ - Agent: a, - Prompt: task, - Dir: tree.Dir, - Stages: stages, - MaxRetries: retries, - Timeout: cfg.TimeoutDuration(), - Memory: nil, // benchmark records to its own table, not run history - Label: "bench:" + a.Name(), - Principles: config.PrinciplesText(cfg.Principles), - }) - results[i].dur = time.Since(start) - results[i].out = out - results[i].err = rerr - if rerr == nil && out.HadChanges { - results[i].files, results[i].added, results[i].removed, _ = mgr.DiffStat(tree) - } - return rerr - }) - - ranked := rankResults(results) - printLeaderboard(ranked) - - // Persist to memory (best-effort). - persistBenchmarks(cmd.Context(), task, ranked) - - // Offer to keep the winner. - winner, ok := firstMergeable(ranked) - if !ok { - fmt.Println(ui.Dim("\nno agent produced a mergeable result — nothing to keep.")) - return nil - } - in := bufio.NewReader(os.Stdin) - q := fmt.Sprintf("\nmerge winner %s into the base tree?", ui.Agent(winner.agent)) - if !winner.valid() && !winner.skipped() { - q = fmt.Sprintf("\nwinner %s did NOT pass validation — merge it anyway?", ui.Agent(winner.agent)) - } - if confirm(in, q) { - conflict, merr := mgr.Merge(winner.tree, "orchestra: benchmark winner ("+winner.agent+"): "+task) - if merr != nil { - return merr - } - if conflict { - fmt.Println(ui.Danger("✗ merge conflict — left unmerged")) - } else { - fmt.Println(ui.Success("✓ merged " + winner.agent + "'s result into the base tree")) - } - } else { - fmt.Println(ui.Dim("discarded — no changes kept.")) - } - return nil + return runBenchmark(cmd.Context(), bufio.NewReader(os.Stdin), cfg, agents, task, jobs, stagesFor(cfg), persistBenchmarks) }, } cmd.Flags().StringVar(&only, "agents", "", "comma-separated agents to benchmark (default: all available)") @@ -188,15 +111,101 @@ func newBenchmarkCmd() *cobra.Command { return cmd } +// runBenchmark runs task through every agent, each in its own isolated tree +// (git worktree or directory copy), prints the leaderboard, hands the ranking +// to persist (may be nil), and offers to merge the winner into flagDir. +func runBenchmark(ctx context.Context, in *bufio.Reader, cfg *config.Config, agents []agent.Agent, task string, jobs int, stages []validate.Stage, persist func(context.Context, string, []benchResult)) error { + retries := cfg.RetryLimit() + + mgr, err := worktree.New(flagDir) + if err != nil { + return err + } + defer mgr.Cleanup() + + names := make([]string, len(agents)) + for i, a := range agents { + names[i] = a.Name() + } + fmt.Printf("%s benchmarking %q across %s\n", ui.Accent("▸"), task, ui.Agent(strings.Join(names, ", "))) + + // Run each agent in its own isolated tree, in parallel. + results := make([]benchResult, len(agents)) + scheduler.Bounded(ctx, jobs, len(agents), func(i int) error { + a := agents[i] + results[i].agent = a.Name() + tree, aerr := mgr.Add("bench-"+a.Name(), "HEAD") + if aerr != nil { + results[i].err = aerr + return aerr + } + results[i].tree = tree + results[i].created = true + + start := time.Now() + out, rerr := engine.ExecuteHeadless(ctx, engine.Options{ + Agent: a, + Prompt: task, + Dir: tree.Dir, + Stages: stages, + MaxRetries: retries, + Timeout: cfg.TimeoutDuration(), + Memory: nil, // benchmark records to its own table, not run history + Label: "bench:" + a.Name(), + Principles: config.PrinciplesText(cfg.Principles), + }) + results[i].dur = time.Since(start) + results[i].out = out + results[i].err = rerr + if rerr == nil && out.HadChanges { + results[i].files, results[i].added, results[i].removed, _ = mgr.DiffStat(tree) + } + return rerr + }) + + ranked := rankResults(results) + printLeaderboard(ranked) + + // Persist to memory (best-effort). + if persist != nil { + persist(ctx, task, ranked) + } + + // Offer to keep the winner. + winner, ok := firstMergeable(ranked) + if !ok { + fmt.Println(ui.Dim("\nno agent produced a mergeable result — nothing to keep.")) + return nil + } + q := fmt.Sprintf("\nmerge winner %s into the base tree?", ui.Agent(winner.agent)) + if !winner.valid() && !winner.skipped() { + q = fmt.Sprintf("\nwinner %s did NOT pass validation — merge it anyway?", ui.Agent(winner.agent)) + } + if confirm(in, q) { + conflict, merr := mgr.Merge(winner.tree, "orchestra: benchmark winner ("+winner.agent+"): "+task) + if merr != nil { + return merr + } + if conflict { + fmt.Println(ui.Danger("✗ merge conflict — left unmerged")) + } else { + fmt.Println(ui.Success("✓ merged " + winner.agent + "'s result into the base tree")) + } + } else { + fmt.Println(ui.Dim("discarded — no changes kept.")) + } + return nil +} + // runCompare runs each agent twice — principles off vs `level` — and reports how // much smaller the lean-code principles make the resulting diff. Pure -// measurement: all worktrees are discarded, nothing is merged. +// measurement: all isolated trees are discarded, nothing is merged. func runCompare(ctx context.Context, cfg *config.Config, agents []agent.Agent, task, level string, jobs int) error { onText := config.PrinciplesText(level) if onText == "" { return fmt.Errorf("--compare needs a non-off principles level (got %q)", level) } - mgr, err := worktree.NewManager(flagDir) + mgr, err := worktree.New(flagDir) if err != nil { return err } diff --git a/cmd/orchestra/do.go b/cmd/orchestra/do.go index 693e114..29e2825 100644 --- a/cmd/orchestra/do.go +++ b/cmd/orchestra/do.go @@ -12,6 +12,7 @@ import ( "github.com/crossben/orchestra-code/internal/agent" "github.com/crossben/orchestra-code/internal/config" "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/fsdiff" "github.com/crossben/orchestra-code/internal/gitutil" "github.com/crossben/orchestra-code/internal/memory" "github.com/crossben/orchestra-code/internal/planner" @@ -36,8 +37,8 @@ func newDoCmd() *cobra.Command { Short: "Plan a request, then execute each step supervised (sequential or parallel)", Long: "Decompose a request into steps, let you approve the plan, then run each step through the\n" + "supervised engine. Sequential by default (one step at a time, halt on rejection). With\n" + - "--parallel, independent steps run concurrently in isolated git worktrees and you review +\n" + - "merge each result.", + "--parallel, independent steps run concurrently in isolation (git worktrees inside a repository,\n" + + "directory copies in a plain folder) and you review + merge each result.", Args: cobra.ArbitraryArgs, RunE: func(cmd *cobra.Command, args []string) error { request := strings.TrimSpace(strings.Join(args, " ")) @@ -56,12 +57,10 @@ func newDoCmd() *cobra.Command { agentName = cfg.DefaultAgent } - // Git pre-flight: sequential mode works anywhere (snapshot-tracked), - // but --parallel isolates steps in git worktrees, so it needs a repo. + // Git pre-flight: both modes work anywhere. Inside a repository the + // tree must start clean; --parallel isolates steps in git worktrees + // there, and in directory copies in a plain folder. inRepo := gitutil.IsRepo(flagDir) - if parallel && !inRepo { - return fmt.Errorf("--parallel needs a git repository (each step runs in an isolated worktree); run inside a repo, or drop --parallel for step-by-step supervision") - } if inRepo { if clean, err := gitutil.IsClean(flagDir); err != nil { return err @@ -122,7 +121,7 @@ func newDoCmd() *cobra.Command { } cmd.Flags().StringVar(&agentName, "agent", "", "agent for planning and implementation (default from config)") cmd.Flags().BoolVar(&yes, "yes", false, "skip the plan-approval prompt") - cmd.Flags().BoolVar(¶llel, "parallel", false, "run independent steps concurrently in isolated worktrees") + cmd.Flags().BoolVar(¶llel, "parallel", false, "run independent steps concurrently in isolated worktrees (or folder copies)") cmd.Flags().IntVar(&jobs, "jobs", 4, "max concurrent steps when --parallel") cmd.Flags().StringVar(&principles, "principles", "", "lean-code principles preamble: off|lite|full (default from config)") return cmd @@ -162,14 +161,16 @@ func runSequential(ctx context.Context, in *bufio.Reader, cfg *config.Config, re } // runParallel executes the plan in dependency waves: every step whose deps are -// merged runs concurrently in its own worktree, then results are reviewed and -// merged one at a time before the next wave unlocks. +// merged runs concurrently in its own isolated tree (a git worktree, or a +// directory copy outside a repository), then results are reviewed and merged +// one at a time before the next wave unlocks. func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg *agent.Registry, ag agent.Agent, agentName string, pl planner.Plan, stages []validate.Stage, jobs int, mem *memory.Store) error { - mgr, err := worktree.NewManager(flagDir) + mgr, err := worktree.New(flagDir) if err != nil { return err } defer mgr.Cleanup() + guard := newBaseGuard(flagDir) nodes := make([]scheduler.Node, len(pl.Steps)) for i, s := range pl.Steps { @@ -201,6 +202,9 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg break } fmt.Printf("\n%s\n", ui.Heading(fmt.Sprintf("── wave %d: %d step(s) in parallel ──", wave, len(ready)))) + if err := guard.arm(); err != nil { + return err + } // Fan-out: run the ready steps concurrently, each in its own worktree. results := make([]result, len(ready)) @@ -230,10 +234,7 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg // worktree (some CLIs don't honor the working directory), the base is now // dirty and merges would fail. Discard that stray work so the properly // isolated branches can still merge cleanly. - if clean, _ := gitutil.IsClean(flagDir); !clean { - fmt.Println(ui.Warn(" ! an agent wrote outside its worktree — discarding stray changes in the base tree")) - _ = gitutil.Restore(flagDir) - } + guard.check() // Fan-in: review + merge each result in order. for k, i := range ready { @@ -343,3 +344,53 @@ func confirm(in *bufio.Reader, question string) bool { return false } } + +// baseGuard catches agents that wrote into the base directory instead of their +// isolated tree (some CLIs don't honor the working directory) and discards that +// stray work. Inside a repository "stray" means the tree is no longer clean; +// in a plain folder it means the folder differs from a snapshot armed before +// the wave's fan-out. +type baseGuard struct { + dir string + inRepo bool + snap *fsdiff.Snapshot +} + +func newBaseGuard(dir string) *baseGuard { + return &baseGuard{dir: dir, inRepo: gitutil.IsRepo(dir)} +} + +// arm records the base state before a wave runs (a no-op inside a repository, +// where the clean tree is the reference). +func (g *baseGuard) arm() error { + if g.inRepo { + return nil + } + snap, err := fsdiff.Capture(g.dir) + if err != nil { + return fmt.Errorf("snapshot %s: %w", g.dir, err) + } + g.snap = snap + return nil +} + +// check discards any change made to the base since arm, reporting whether it did. +func (g *baseGuard) check() bool { + if g.inRepo { + if clean, _ := gitutil.IsClean(g.dir); !clean { + fmt.Println(ui.Warn(" ! an agent wrote outside its worktree — discarding stray changes in the base tree")) + _ = gitutil.Restore(g.dir) + return true + } + return false + } + if g.snap == nil { + return false + } + if d, err := fsdiff.Diff(g.dir, g.snap); err != nil || d == "" { + return false + } + fmt.Println(ui.Warn(" ! an agent wrote outside its isolated copy — discarding stray changes in the base folder")) + _ = fsdiff.Restore(g.dir, g.snap) + return true +} diff --git a/cmd/orchestra/gitfree_test.go b/cmd/orchestra/gitfree_test.go new file mode 100644 index 0000000..6c3c70a --- /dev/null +++ b/cmd/orchestra/gitfree_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "bufio" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/config" + "github.com/crossben/orchestra-code/internal/planner" +) + +// fakeAgent writes a shell script and registers it as a CLI agent. The script +// runs in the task directory (or is told it via dirFlag) with the prompt as its +// last argument — exactly how real agent CLIs are driven. +func fakeAgent(t *testing.T, reg *agent.Registry, name, dirFlag, script string) agent.Agent { + t.Helper() + bin := filepath.Join(t.TempDir(), name+".sh") + if err := os.WriteFile(bin, []byte("#!/bin/sh\n"+script+"\n"), 0o755); err != nil { + t.Fatal(err) + } + a := agent.New(name, bin, nil, dirFlag, []agent.Capability{agent.CapImplement}) + reg.Add(a) + return a +} + +// plainFolder returns a non-git temp dir with one file, set as --dir for the +// duration of the test. +func plainFolder(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "base.txt"), []byte("base\n"), 0o644); err != nil { + t.Fatal(err) + } + old := flagDir + flagDir = dir + t.Cleanup(func() { flagDir = old }) + return dir +} + +func fileIs(t *testing.T, path, want string) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if string(b) != want { + t.Fatalf("%s = %q, want %q", path, b, want) + } +} + +func twoStepPlan() planner.Plan { + return planner.Plan{Steps: []planner.Step{ + {Title: "step a", Agent: "alpha"}, + {Title: "step b", Agent: "beta"}, + }} +} + +func TestParallelPlainFolderMergesBoth(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + alpha := fakeAgent(t, reg, "alpha", "", `printf 'from alpha\n' > a.txt`) + // beta ignores cwd and is told its directory through dir_flag ("$2"). + fakeAgent(t, reg, "beta", "--dir", `cd /; mkdir -p "$2/pkg" && printf 'from beta\n' > "$2/pkg/b.txt"`) + + in := bufio.NewReader(strings.NewReader("y\ny\n")) + if err := runParallel(context.Background(), in, config.Default(), reg, alpha, "alpha", twoStepPlan(), nil, 2, nil); err != nil { + t.Fatalf("runParallel: %v", err) + } + fileIs(t, filepath.Join(dir, "a.txt"), "from alpha\n") + fileIs(t, filepath.Join(dir, "pkg", "b.txt"), "from beta\n") + fileIs(t, filepath.Join(dir, "base.txt"), "base\n") + if _, err := os.Stat(filepath.Join(dir, ".git")); !os.IsNotExist(err) { + t.Fatal("a plain folder must not become a git repository") + } +} + +func TestParallelPlainFolderConflict(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + alpha := fakeAgent(t, reg, "alpha", "", `printf 'alpha\n' > base.txt; printf 'a\n' > a-only.txt`) + fakeAgent(t, reg, "beta", "", `printf 'beta\n' > base.txt; printf 'b\n' > b-only.txt`) + + in := bufio.NewReader(strings.NewReader("y\ny\n")) + if err := runParallel(context.Background(), in, config.Default(), reg, alpha, "alpha", twoStepPlan(), nil, 2, nil); err != nil { + t.Fatalf("runParallel: %v", err) + } + // Step 1 (alpha) merges first; step 2 touched the same file → conflict, + // and nothing of step 2 lands (all-or-nothing). + fileIs(t, filepath.Join(dir, "base.txt"), "alpha\n") + fileIs(t, filepath.Join(dir, "a-only.txt"), "a\n") + if _, err := os.Stat(filepath.Join(dir, "b-only.txt")); !os.IsNotExist(err) { + t.Fatal("a conflicting step must not write any of its files") + } +} + +func TestParallelPlainFolderRejectKeepsNothing(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + alpha := fakeAgent(t, reg, "alpha", "", `printf 'a\n' > a.txt`) + fakeAgent(t, reg, "beta", "", `printf 'b\n' > b.txt`) + + in := bufio.NewReader(strings.NewReader("n\ny\n")) // reject step 1, accept step 2 + if err := runParallel(context.Background(), in, config.Default(), reg, alpha, "alpha", twoStepPlan(), nil, 2, nil); err != nil { + t.Fatalf("runParallel: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "a.txt")); !os.IsNotExist(err) { + t.Fatal("a rejected step must leave no files behind") + } + fileIs(t, filepath.Join(dir, "b.txt"), "b\n") +} + +func TestParallelPlainFolderDiscardsStrayWrites(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + // alpha does its job in its copy but also scribbles into the base folder. + alpha := fakeAgent(t, reg, "alpha", "", `printf 'a\n' > a.txt; printf 'stray\n' > '`+filepath.Join(dir, "stray.txt")+`'`) + fakeAgent(t, reg, "beta", "", `true`) + + in := bufio.NewReader(strings.NewReader("y\n")) + if err := runParallel(context.Background(), in, config.Default(), reg, alpha, "alpha", twoStepPlan(), nil, 2, nil); err != nil { + t.Fatalf("runParallel: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "stray.txt")); !os.IsNotExist(err) { + t.Fatal("stray write into the base folder should have been discarded") + } + fileIs(t, filepath.Join(dir, "a.txt"), "a\n") +} + +func TestBenchmarkPlainFolderMergesWinner(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + alpha := fakeAgent(t, reg, "alpha", "", `printf 'alpha wins\n' > win.txt`) + beta := fakeAgent(t, reg, "beta", "", `true`) // makes no changes + + var ranked []benchResult + persist := func(_ context.Context, _ string, r []benchResult) { ranked = r } + in := bufio.NewReader(strings.NewReader("y\n")) + if err := runBenchmark(context.Background(), in, config.Default(), []agent.Agent{alpha, beta}, "write win.txt", 2, nil, persist); err != nil { + t.Fatalf("runBenchmark: %v", err) + } + if len(ranked) != 2 || ranked[0].agent != "alpha" { + t.Fatalf("ranking = %+v, want alpha first", ranked) + } + if r := ranked[0]; r.err != nil || !r.out.HadChanges || r.files != 1 || r.added != 1 || r.removed != 0 { + t.Fatalf("alpha result = err %v changes %v files %d +%d -%d", r.err, r.out.HadChanges, r.files, r.added, r.removed) + } + fileIs(t, filepath.Join(dir, "win.txt"), "alpha wins\n") +} + +func TestBenchmarkPlainFolderDeclineKeepsNothing(t *testing.T) { + dir := plainFolder(t) + reg := agent.NewRegistry() + alpha := fakeAgent(t, reg, "alpha", "", `printf 'x\n' > win.txt`) + + in := bufio.NewReader(strings.NewReader("n\n")) + if err := runBenchmark(context.Background(), in, config.Default(), []agent.Agent{alpha}, "t", 1, nil, nil); err != nil { + t.Fatalf("runBenchmark: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "win.txt")); !os.IsNotExist(err) { + t.Fatal("declining the winner must keep nothing") + } +} From 3e92473a526be77c767c4e9c5f40f7bc5cef3721 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:02:15 +0000 Subject: [PATCH 14/25] docs: parallel and benchmark work in plain folders Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++++++++++++++++-------- docs/EXTENDING.md | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 093a9ab..1412259 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,9 @@ Orchestra dispatches coding-agent CLIs (Claude Code, OpenCode, Mimo, …) through one **supervised** interface. You just chat: it **answers plain questions, routes coding tasks to the best agent**, validates every result (build → lint → test), lets the agent fix its own failures, and keeps nothing without your `y`. -It can **decompose a big request into steps** — running independent ones **in parallel across isolated git -worktrees** — remembers every run, and can **benchmark agents** against each other. +It can **decompose a big request into steps** — running independent ones **in parallel, each isolated in +its own git worktree (or folder copy outside a repo)** — remembers every run, and can **benchmark agents** +against each other. ```sh # install (Linux/macOS) — prebuilt binary @@ -90,8 +91,9 @@ orchestra do "build user authentication" # plan → approve → run each ste `do` commits each accepted step and **halts at the first rejected step** (prior steps stay committed). **Parallel** (`--parallel`): the planner marks which steps are independent; Orchestra runs each ready -step concurrently in its own **git worktree**, then you review + merge each result before the next -dependency wave unlocks: +step concurrently in its own **git worktree** (or, in a folder that isn't a git repository, its own +**temporary copy of the folder**), then you review + merge each result before the next dependency wave +unlocks: ```sh orchestra do --parallel --jobs 4 "build the API, the CLI, and the docs" @@ -102,6 +104,12 @@ branch is merged into the base with **conflict detection** (a conflicting merge dependents are skipped); rejected branches are discarded. The base working tree is never touched during execution — all work happens in isolated worktrees. +Outside a git repository the copies are made with the same ignore rules as snapshot diffs (dependency and +build directories such as `node_modules`, `vendor`, `dist` are not copied), and an accepted step's files are +written back only if none of them changed in your folder since the step started — otherwise the step is +reported as a conflict and **nothing** of it is written. The check is per file: two steps editing the same +file conflict even if they touched different lines. Copies are deleted when the run ends. + ### Memory & history Every run is recorded in `~/.orchestra/orchestra.db` (outside your repo, so it never dirties the tree): @@ -143,7 +151,8 @@ Colors adapt to light and dark terminals. ### Benchmark agents -Run the **same task through every agent** (each isolated in its own worktree, in parallel) and rank them: +Run the **same task through every agent** (each isolated in its own worktree — or folder copy outside a +git repository — in parallel) and rank them: ```sh orchestra benchmark "add input validation to the login form" @@ -244,8 +253,8 @@ agents: ``` > **`dir_flag`** tells Orchestra how to pass an agent its working directory. Most CLIs honor the process -> cwd, but some (e.g. opencode) don't — set `dir_flag` so parallel worktrees stay isolated. Orchestra -> also guards the base tree: if an agent writes outside its worktree anyway, the stray changes are +> cwd, but some (e.g. opencode) don't — set `dir_flag` so parallel worktrees (and folder copies) stay +> isolated. Orchestra also guards the base tree: if an agent writes outside its worktree anyway, the stray changes are > discarded before merge so one misbehaving agent can't break the wave. A config file overrides defaults and adds agents; matching names replace the built-in entry. @@ -280,7 +289,7 @@ internal/tui Bubble Tea dashboard: agents / history / benchmarks (read-o internal/router AI routing: Classifier (CLI now, API later) → Decision, 3-tier fallback internal/planner decompose a request into ordered steps (+ depends_on for parallel) internal/scheduler bounded-concurrency runner + DAG waves (cycle/blocked detection) -internal/worktree git-worktree isolation: branch per task, merge + conflict detection +internal/worktree task isolation: git worktrees in a repo, folder copies elsewhere; merge + conflict detection internal/engine supervised pipeline (dispatch → validate → retry → review) + headless mode ← run/shell/do internal/shell interactive chat REPL internal/memory SQLite run history + preferred-agent hint (~/.orchestra) diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 4507d05..967b0c4 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -90,7 +90,8 @@ Notes: - Self-correction works like any agent: on validation failure the failure text is fed back and the model re-edits its own prior changes in place. - Works everywhere CLI agents do: `run`, shell, dashboard chat, planning, routing, and even - `do --parallel` worktrees (patches apply inside each isolated worktree). + `do --parallel` worktrees (patches apply inside each isolated worktree, or folder copy + outside a git repository). --- From 5dd3b4187b99690a0b7add6c9135fed49c855a1d Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:02:23 +0000 Subject: [PATCH 15/25] engine/memory: sum usage across attempts and persist tokens/cost Outcome and Turn carry Usage summed over every attempt (including one whose agent call failed); EventAgentDone carries per-attempt usage for the dashboard. run/do/shell print a one-line summary when usage is known. runs and benchmarks gain tokens_in, tokens_out and cost_usd via the idempotent ADD COLUMN migration; StatsByAgent returns per-agent totals. Co-Authored-By: Claude Opus 5.5 --- internal/agent/usage.go | 18 ++++++ internal/agent/usage_test.go | 12 ++++ internal/engine/engine.go | 31 ++++++--- internal/engine/usage_test.go | 115 +++++++++++++++++++++++++++++++++ internal/memory/memory.go | 83 ++++++++++++++++++------ internal/memory/memory_test.go | 108 +++++++++++++++++++++++++++++++ 6 files changed, 337 insertions(+), 30 deletions(-) create mode 100644 internal/engine/usage_test.go diff --git a/internal/agent/usage.go b/internal/agent/usage.go index 6b50b89..26ff6ac 100644 --- a/internal/agent/usage.go +++ b/internal/agent/usage.go @@ -61,6 +61,24 @@ func FormatTokens(n int) string { } } +// TokensCell is the compact table cell for stored usage: "12.3k/1.1k" (in/out), +// or "—" when nothing was reported (stored zero = unknown). +func TokensCell(in, out int) string { + if in == 0 && out == 0 { + return "—" + } + return FormatTokens(in) + "/" + FormatTokens(out) +} + +// CostCell is the compact table cell for a stored cost, "—" when not priced +// (stored zero). +func CostCell(usd float64) string { + if usd <= 0 { + return "—" + } + return FormatCost(usd) +} + // FormatCost renders USD with cents, or four decimals below one cent so a // cheap run does not read as free. func FormatCost(usd float64) string { diff --git a/internal/agent/usage_test.go b/internal/agent/usage_test.go index 7ea44f8..e057f31 100644 --- a/internal/agent/usage_test.go +++ b/internal/agent/usage_test.go @@ -63,6 +63,18 @@ func TestFormatTokensAndCost(t *testing.T) { } } +func TestTableCells(t *testing.T) { + if TokensCell(0, 0) != "—" || CostCell(0) != "—" { + t.Fatal("unknown usage should render as —") + } + if got := TokensCell(12_345, 1_100); got != "12.3k/1.1k" { + t.Fatalf("TokensCell = %q", got) + } + if got := CostCell(0.042); got != "$0.04" { + t.Fatalf("CostCell = %q", got) + } +} + // usageProvider is a fake llm.Provider that reports token usage. type usageProvider struct { fakeProvider diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 21d4476..6c5ccf7 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -64,7 +64,7 @@ type EventKind int const ( EventAttempt EventKind = iota // an attempt is starting (Attempt, Max) - EventAgentDone // the agent exited (ExitCode, Duration) + EventAgentDone // the agent exited (ExitCode, Duration, Usage) EventStageStart // a validation stage started (Stage) EventStageDone // a validation stage finished (Stage, Passed) ) @@ -78,6 +78,7 @@ type Event struct { Passed bool ExitCode int Duration time.Duration + Usage agent.Usage // EventAgentDone: this attempt's tokens/cost (zero when unreported) } func (o Options) emit(e Event) { @@ -101,7 +102,8 @@ type Outcome struct { Report validate.Report HadChanges bool Accepted bool - Diff string // the reviewed diff ("" when nothing changed) + Diff string // the reviewed diff ("" when nothing changed) + Usage agent.Usage // tokens/cost summed over every attempt } // Execute runs the full supervised pipeline once (including retries). The reader @@ -257,6 +259,7 @@ type Turn struct { Diff string HadChanges bool Err error + Usage agent.Usage // tokens/cost summed over every attempt // Baseline is the pre-turn directory snapshot when the working dir is not // a git repository (nil inside a repo). The TUI passes it back to Revert. @@ -306,13 +309,14 @@ func Produce(ctx context.Context, opts Options) Turn { Timeout: opts.Timeout, Output: opts.Output, }) + t.Usage = t.Usage.Add(res.Usage) // spent even when the attempt failed if err != nil { t.Err = err return t } t.ExitCode = res.ExitCode t.AgentText = res.Output - opts.emit(Event{Kind: EventAgentDone, Attempt: attempt, Max: maxAttempts, ExitCode: res.ExitCode, Duration: res.Duration}) + opts.emit(Event{Kind: EventAgentDone, Attempt: attempt, Max: maxAttempts, ExitCode: res.ExitCode, Duration: res.Duration, Usage: res.Usage}) t.Report = validate.RunPipelineObserved(ctx, opts.Dir, opts.Stages, func(done bool, r validate.StageResult) { if done { opts.emit(Event{Kind: EventStageDone, Attempt: attempt, Max: maxAttempts, Stage: r.Name, Passed: r.Passed}) @@ -365,6 +369,7 @@ func runLoop(ctx context.Context, opts Options) (out Outcome, err error) { if opts.Label == "" { fmt.Println(ui.Rule(48)) } + out.Usage = out.Usage.Add(res.Usage) // spent even when the attempt failed if rerr != nil { return out, fmt.Errorf("agent %q failed to run: %w", opts.Agent.Name(), rerr) } @@ -385,6 +390,9 @@ func runLoop(ctx context.Context, opts Options) (out Outcome, err error) { } prompt = retryPrompt(opts.Prompt, out.Report) } + if u := out.Usage.String(); u != "" { + opts.logf("%s %s", ui.Accent("▸"), ui.Dim(u)) + } return out, nil } @@ -429,13 +437,16 @@ func recordMemory(opts Options, out Outcome, outcome string) { dir = abs } if rerr := opts.Memory.Record(memory.Run{ - Dir: dir, - Agent: opts.Agent.Name(), - Prompt: opts.Prompt, - Outcome: outcome, - Attempts: out.Attempts, - Passed: out.Report.Passed(), - Diff: out.Diff, + Dir: dir, + Agent: opts.Agent.Name(), + Prompt: opts.Prompt, + Outcome: outcome, + Attempts: out.Attempts, + Passed: out.Report.Passed(), + Diff: out.Diff, + TokensIn: out.Usage.InputTokens, + TokensOut: out.Usage.OutputTokens, + CostUSD: out.Usage.CostUSD, }, time.Now()); rerr != nil { opts.logf("(warning: could not record to memory: %v)", rerr) } diff --git a/internal/engine/usage_test.go b/internal/engine/usage_test.go new file mode 100644 index 0000000..70839e5 --- /dev/null +++ b/internal/engine/usage_test.go @@ -0,0 +1,115 @@ +package engine + +import ( + "bufio" + "context" + "errors" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/memory" + "github.com/crossben/orchestra-code/internal/validate" +) + +// meteredAgent is an in-process fake API-style agent: every attempt reports +// the same token usage; the first attempt writes a failing file, later ones +// fix it (so validation forces exactly one retry). +type meteredAgent struct { + calls int + usage agent.Usage + fail error // returned (with usage) on every call when set +} + +func (m *meteredAgent) Name() string { return "metered" } +func (m *meteredAgent) Health() error { return nil } +func (m *meteredAgent) Capabilities() []agent.Capability { + return []agent.Capability{agent.CapImplement} +} +func (m *meteredAgent) Run(_ context.Context, task agent.Task) (agent.Result, error) { + m.calls++ + if m.fail != nil { + return agent.Result{Usage: m.usage}, m.fail + } + content := "broken\n" + if m.calls >= 2 { + content = "fixed\n" + } + if err := os.WriteFile(filepath.Join(task.Dir, "out.txt"), []byte(content), 0o644); err != nil { + return agent.Result{}, err + } + return agent.Result{Output: "ok", Usage: m.usage}, nil +} + +var attemptUsage = agent.Usage{InputTokens: 1000, OutputTokens: 100, CostUSD: 0.0045, Known: true, Priced: true} + +var fixedCheck = []validate.Stage{{Name: "test", Command: "grep -q fixed out.txt"}} + +func TestProduceSumsUsageAcrossRetries(t *testing.T) { + var perAttempt []agent.Usage + turn := Produce(context.Background(), Options{ + Agent: &meteredAgent{usage: attemptUsage}, Prompt: "fix", Dir: t.TempDir(), + Stages: fixedCheck, MaxRetries: 2, + OnEvent: func(e Event) { + if e.Kind == EventAgentDone { + perAttempt = append(perAttempt, e.Usage) + } + }, + }) + if turn.Err != nil || turn.Attempts != 2 { + t.Fatalf("turn: attempts=%d err=%v", turn.Attempts, turn.Err) + } + u := turn.Usage + if !u.Known || !u.Priced || u.InputTokens != 2000 || u.OutputTokens != 200 || math.Abs(u.CostUSD-0.009) > 1e-12 { + t.Fatalf("summed usage = %+v", u) + } + if len(perAttempt) != 2 || perAttempt[0] != attemptUsage || perAttempt[1] != attemptUsage { + t.Fatalf("EventAgentDone should carry each attempt's usage: %+v", perAttempt) + } +} + +// Tokens spent on an attempt whose agent call failed still count. +func TestProduceKeepsUsageOnAgentError(t *testing.T) { + turn := Produce(context.Background(), Options{ + Agent: &meteredAgent{usage: attemptUsage, fail: errors.New("apply failed")}, Prompt: "x", Dir: t.TempDir(), + }) + if turn.Err == nil || turn.Usage != attemptUsage { + t.Fatalf("err=%v usage=%+v", turn.Err, turn.Usage) + } +} + +func TestExecuteSumsUsageAndRecordsIt(t *testing.T) { + mem, err := memory.Open(filepath.Join(t.TempDir(), "o.db")) + if err != nil { + t.Fatal(err) + } + defer mem.Close() + out, err := Execute(context.Background(), bufio.NewReader(strings.NewReader("y\n")), Options{ + Agent: &meteredAgent{usage: attemptUsage}, Prompt: "fix", Dir: t.TempDir(), + Stages: fixedCheck, MaxRetries: 2, Memory: mem, + }) + if err != nil { + t.Fatal(err) + } + if out.Attempts != 2 || out.Usage.InputTokens != 2000 || out.Usage.OutputTokens != 200 { + t.Fatalf("outcome usage: attempts=%d %+v", out.Attempts, out.Usage) + } + runs, err := mem.Recent("", 5) + if err != nil || len(runs) != 1 { + t.Fatalf("runs=%+v err=%v", runs, err) + } + if r := runs[0]; r.TokensIn != 2000 || r.TokensOut != 200 || math.Abs(r.CostUSD-0.009) > 1e-12 { + t.Fatalf("recorded run usage: %+v", r) + } +} + +// CLI-style agents report nothing: usage stays unknown end to end. +func TestProduceUnknownUsageForCLIAgents(t *testing.T) { + turn := Produce(context.Background(), Options{Agent: noopAgent{}, Prompt: "x", Dir: t.TempDir()}) + if turn.Usage.Known || turn.Usage.String() != "" { + t.Fatalf("usage should be unknown: %+v", turn.Usage) + } +} diff --git a/internal/memory/memory.go b/internal/memory/memory.go index a4cab87..9d3ac8e 100644 --- a/internal/memory/memory.go +++ b/internal/memory/memory.go @@ -34,6 +34,12 @@ type Run struct { Attempts int Passed bool // validation passed Diff string // unified diff the run produced ("" when none or not recorded) + + // Token usage summed over every attempt. Zero tokens = unknown (CLI + // agents); zero cost = not priced. + TokensIn int + TokensOut int + CostUSD float64 } // DefaultPath returns ~/.orchestra/orchestra.db. @@ -97,14 +103,32 @@ CREATE INDEX IF NOT EXISTS idx_bench_dir ON benchmarks(dir);`) if err != nil { return err } - // Added after the first release: databases created earlier lack the diff - // column. SQLite has no ADD COLUMN IF NOT EXISTS, so check first. - has, err := s.hasColumn("runs", "diff") - if err != nil || has { - return err + // Columns added after the first release. Databases created earlier lack + // them; SQLite has no ADD COLUMN IF NOT EXISTS, so check each first. + for _, c := range addedColumns { + has, err := s.hasColumn(c.table, c.name) + if err != nil { + return err + } + if has { + continue + } + if _, err := s.db.Exec(`ALTER TABLE ` + c.table + ` ADD COLUMN ` + c.name + ` ` + c.def); err != nil { + return fmt.Errorf("add column %s.%s: %w", c.table, c.name, err) + } } - _, err = s.db.Exec(`ALTER TABLE runs ADD COLUMN diff TEXT NOT NULL DEFAULT ''`) - return err + return nil +} + +// addedColumns are applied idempotently, in order, on every Open. +var addedColumns = []struct{ table, name, def string }{ + {"runs", "diff", `TEXT NOT NULL DEFAULT ''`}, + {"runs", "tokens_in", `INTEGER NOT NULL DEFAULT 0`}, + {"runs", "tokens_out", `INTEGER NOT NULL DEFAULT 0`}, + {"runs", "cost_usd", `REAL NOT NULL DEFAULT 0`}, + {"benchmarks", "tokens_in", `INTEGER NOT NULL DEFAULT 0`}, + {"benchmarks", "tokens_out", `INTEGER NOT NULL DEFAULT 0`}, + {"benchmarks", "cost_usd", `REAL NOT NULL DEFAULT 0`}, } // hasColumn reports whether table has a column named col. @@ -141,6 +165,10 @@ type BenchRun struct { Removed int Exit int Won bool + + TokensIn int + TokensOut int + CostUSD float64 } // RecordBenchmark stores one agent's benchmark result. @@ -152,11 +180,11 @@ func (s *Store) RecordBenchmark(r BenchRun, now time.Time) error { return 0 } _, err := s.db.Exec( - `INSERT INTO benchmarks (ts, dir, task, agent, valid, skipped, changed, duration_ms, retries, files, added, removed, exit, won) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO benchmarks (ts, dir, task, agent, valid, skipped, changed, duration_ms, retries, files, added, removed, exit, won, tokens_in, tokens_out, cost_usd) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, now.UTC().Format(time.RFC3339), r.Dir, r.Task, r.Agent, b(r.Valid), b(r.Skipped), b(r.Changed), r.Duration.Milliseconds(), r.Retries, - r.Files, r.Added, r.Removed, r.Exit, b(r.Won), + r.Files, r.Added, r.Removed, r.Exit, b(r.Won), r.TokensIn, r.TokensOut, r.CostUSD, ) return err } @@ -168,8 +196,10 @@ func (s *Store) Record(r Run, now time.Time) error { passed = 1 } _, err := s.db.Exec( - `INSERT INTO runs (ts, dir, agent, prompt, outcome, attempts, passed, diff) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO runs (ts, dir, agent, prompt, outcome, attempts, passed, diff, tokens_in, tokens_out, cost_usd) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, now.UTC().Format(time.RFC3339), r.Dir, r.Agent, r.Prompt, r.Outcome, r.Attempts, passed, r.Diff, + r.TokensIn, r.TokensOut, r.CostUSD, ) return err } @@ -180,10 +210,11 @@ func (s *Store) Recent(dir string, limit int) ([]Run, error) { rows *sql.Rows err error ) + const cols = `id, ts, dir, agent, prompt, outcome, attempts, passed, diff, tokens_in, tokens_out, cost_usd` if dir == "" { - rows, err = s.db.Query(`SELECT id, ts, dir, agent, prompt, outcome, attempts, passed, diff FROM runs ORDER BY id DESC LIMIT ?`, limit) + rows, err = s.db.Query(`SELECT `+cols+` FROM runs ORDER BY id DESC LIMIT ?`, limit) } else { - rows, err = s.db.Query(`SELECT id, ts, dir, agent, prompt, outcome, attempts, passed, diff FROM runs WHERE dir = ? ORDER BY id DESC LIMIT ?`, dir, limit) + rows, err = s.db.Query(`SELECT `+cols+` FROM runs WHERE dir = ? ORDER BY id DESC LIMIT ?`, dir, limit) } if err != nil { return nil, err @@ -195,7 +226,8 @@ func (s *Store) Recent(dir string, limit int) ([]Run, error) { var r Run var ts string var passed int - if err := rows.Scan(&r.ID, &ts, &r.Dir, &r.Agent, &r.Prompt, &r.Outcome, &r.Attempts, &passed, &r.Diff); err != nil { + if err := rows.Scan(&r.ID, &ts, &r.Dir, &r.Agent, &r.Prompt, &r.Outcome, &r.Attempts, &passed, &r.Diff, + &r.TokensIn, &r.TokensOut, &r.CostUSD); err != nil { return nil, err } r.Time, _ = time.Parse(time.RFC3339, ts) @@ -229,12 +261,18 @@ type AgentStats struct { Runs int Accepted int LastUsed time.Time + + // Usage totals across the agent's runs (zero when never reported/priced). + TokensIn int + TokensOut int + CostUSD float64 } -// StatsByAgent returns per-agent run counts, accepted counts and last-used time -// for dir (all dirs if ""). +// StatsByAgent returns per-agent run counts, accepted counts, last-used time +// and token/cost totals for dir (all dirs if ""). func (s *Store) StatsByAgent(dir string) (map[string]AgentStats, error) { - const q = `SELECT agent, COUNT(*), SUM(outcome = 'accepted'), MAX(ts) FROM runs` + const q = `SELECT agent, COUNT(*), SUM(outcome = 'accepted'), MAX(ts), + SUM(tokens_in), SUM(tokens_out), SUM(cost_usd) FROM runs` var ( rows *sql.Rows err error @@ -255,7 +293,7 @@ func (s *Store) StatsByAgent(dir string) (map[string]AgentStats, error) { st AgentStats ts string ) - if err := rows.Scan(&name, &st.Runs, &st.Accepted, &ts); err != nil { + if err := rows.Scan(&name, &st.Runs, &st.Accepted, &ts, &st.TokensIn, &st.TokensOut, &st.CostUSD); err != nil { return nil, err } st.LastUsed, _ = time.Parse(time.RFC3339, ts) @@ -276,6 +314,10 @@ type BenchRow struct { Files int Added int Removed int + + TokensIn int + TokensOut int + CostUSD float64 } // RecentBenchmarks returns recent benchmark rows for dir (all dirs if ""). @@ -284,7 +326,7 @@ func (s *Store) RecentBenchmarks(dir string, limit int) ([]BenchRow, error) { rows *sql.Rows err error ) - const cols = `ts, task, agent, valid, won, duration_ms, retries, files, added, removed` + const cols = `ts, task, agent, valid, won, duration_ms, retries, files, added, removed, tokens_in, tokens_out, cost_usd` if dir == "" { rows, err = s.db.Query(`SELECT `+cols+` FROM benchmarks ORDER BY id DESC LIMIT ?`, limit) } else { @@ -304,7 +346,8 @@ func (s *Store) RecentBenchmarks(dir string, limit int) ([]BenchRow, error) { won int durationMs int64 ) - if err := rows.Scan(&ts, &r.Task, &r.Agent, &valid, &won, &durationMs, &r.Retries, &r.Files, &r.Added, &r.Removed); err != nil { + if err := rows.Scan(&ts, &r.Task, &r.Agent, &valid, &won, &durationMs, &r.Retries, &r.Files, &r.Added, &r.Removed, + &r.TokensIn, &r.TokensOut, &r.CostUSD); err != nil { return nil, err } r.Time, _ = time.Parse(time.RFC3339, ts) diff --git a/internal/memory/memory_test.go b/internal/memory/memory_test.go index 567250e..025154a 100644 --- a/internal/memory/memory_test.go +++ b/internal/memory/memory_test.go @@ -2,6 +2,7 @@ package memory import ( "database/sql" + "math" "path/filepath" "testing" "time" @@ -66,3 +67,110 @@ INSERT INTO runs (ts, dir, agent, prompt, outcome, attempts, passed) VALUES ('20 } } } + +func TestUsageRoundTripAndAgentTotals(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "o.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + now := time.Now() + for _, r := range []Run{ + {Dir: "/p", Agent: "api", Prompt: "one", Outcome: "accepted", Attempts: 1, TokensIn: 12_000, TokensOut: 800, CostUSD: 0.05}, + {Dir: "/p", Agent: "api", Prompt: "two", Outcome: "rejected", Attempts: 2, TokensIn: 3_000, TokensOut: 200, CostUSD: 0.01}, + {Dir: "/p", Agent: "cli", Prompt: "three", Outcome: "accepted", Attempts: 1}, + } { + if err := s.Record(r, now); err != nil { + t.Fatal(err) + } + } + runs, err := s.Recent("/p", 10) + if err != nil { + t.Fatal(err) + } + if r := runs[2]; r.TokensIn != 12_000 || r.TokensOut != 800 || r.CostUSD != 0.05 { + t.Fatalf("run usage not round-tripped: %+v", r) + } + if r := runs[0]; r.TokensIn != 0 || r.CostUSD != 0 { + t.Fatalf("cli run should have no usage: %+v", r) + } + + stats, err := s.StatsByAgent("/p") + if err != nil { + t.Fatal(err) + } + if st := stats["api"]; st.TokensIn != 15_000 || st.TokensOut != 1_000 || math.Abs(st.CostUSD-0.06) > 1e-9 { + t.Fatalf("api totals: %+v", st) + } + if st := stats["cli"]; st.TokensIn != 0 || st.CostUSD != 0 { + t.Fatalf("cli totals: %+v", st) + } + + if err := s.RecordBenchmark(BenchRun{Dir: "/p", Task: "t", Agent: "api", Valid: true, Won: true, + TokensIn: 4_000, TokensOut: 300, CostUSD: 0.02}, now); err != nil { + t.Fatal(err) + } + benches, err := s.RecentBenchmarks("/p", 5) + if err != nil || len(benches) != 1 { + t.Fatalf("benches=%+v err=%v", benches, err) + } + if b := benches[0]; b.TokensIn != 4_000 || b.TokensOut != 300 || b.CostUSD != 0.02 { + t.Fatalf("bench usage not round-tripped: %+v", b) + } +} + +// A database from before usage tracking (runs without usage columns, and a +// benchmarks table without them) upgrades in place, idempotently, and old rows +// read back as zero usage. +func TestMigrateAddsUsageColumnsToOldSchema(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, dir TEXT NOT NULL, agent TEXT NOT NULL, + prompt TEXT NOT NULL, outcome TEXT NOT NULL, attempts INTEGER NOT NULL, passed INTEGER NOT NULL, + diff TEXT NOT NULL DEFAULT ''); +CREATE TABLE benchmarks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, dir TEXT NOT NULL, task TEXT NOT NULL, + agent TEXT NOT NULL, valid INTEGER NOT NULL, skipped INTEGER NOT NULL, changed INTEGER NOT NULL, + duration_ms INTEGER NOT NULL, retries INTEGER NOT NULL, files INTEGER NOT NULL, added INTEGER NOT NULL, + removed INTEGER NOT NULL, exit INTEGER NOT NULL, won INTEGER NOT NULL); +INSERT INTO runs (ts, dir, agent, prompt, outcome, attempts, passed) VALUES ('2026-01-01T00:00:00Z', '/p', 'a', 'old', 'accepted', 1, 1); +INSERT INTO benchmarks (ts, dir, task, agent, valid, skipped, changed, duration_ms, retries, files, added, removed, exit, won) + VALUES ('2026-01-01T00:00:00Z', '/p', 'old task', 'a', 1, 0, 1, 1500, 0, 1, 2, 1, 0, 1);`); err != nil { + t.Fatal(err) + } + db.Close() + + for i := 0; i < 2; i++ { + s, err := Open(path) + if err != nil { + t.Fatalf("open #%d: %v", i+1, err) + } + runs, rerr := s.Recent("/p", 10) + benches, berr := s.RecentBenchmarks("/p", 10) + stats, serr := s.StatsByAgent("/p") + if rerr != nil || berr != nil || serr != nil { + t.Fatalf("open #%d: %v %v %v", i+1, rerr, berr, serr) + } + if len(runs) != 1 || runs[0].TokensIn != 0 || runs[0].CostUSD != 0 { + t.Fatalf("open #%d: runs=%+v", i+1, runs) + } + if len(benches) != 1 || benches[0].Task != "old task" || benches[0].TokensOut != 0 { + t.Fatalf("open #%d: benches=%+v", i+1, benches) + } + if stats["a"].Runs != 1 || stats["a"].TokensIn != 0 { + t.Fatalf("open #%d: stats=%+v", i+1, stats) + } + // New rows with usage can be written into the upgraded tables. + if err := s.Record(Run{Dir: "/q", Agent: "b", Prompt: "new", Outcome: "accepted", Attempts: 1, TokensIn: 10, TokensOut: 5, CostUSD: 0.001}, time.Now()); err != nil { + t.Fatalf("open #%d: record: %v", i+1, err) + } + if err := s.RecordBenchmark(BenchRun{Dir: "/q", Task: "new", Agent: "b", TokensIn: 10, TokensOut: 5, CostUSD: 0.001}, time.Now()); err != nil { + t.Fatalf("open #%d: record benchmark: %v", i+1, err) + } + s.Close() + } +} From 23f25b5049345c3e8dd37ee69392b6f9bac04c5b Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:02:28 +0000 Subject: [PATCH 16/25] cli: default version to dev so local builds never self-update Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/orchestra/root.go b/cmd/orchestra/root.go index ffbb6aa..5acb6d1 100644 --- a/cmd/orchestra/root.go +++ b/cmd/orchestra/root.go @@ -12,10 +12,10 @@ import ( "github.com/spf13/cobra" ) -// version is the build version. It defaults to the last released tag for -// `go install`, and is overridden at release time via -// -ldflags "-X main.version=" (see .goreleaser.yaml). -var version = "0.8.0" +// version is the build version, stamped at release time via +// -ldflags "-X main.version=" (see .goreleaser.yaml). Local and +// `go install` builds report "dev", which the self-updater never replaces. +var version = "dev" // persistent flags shared across subcommands. var ( From cf781c06b8b1a9c9ade585df1a2d7654217ec549 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:04:09 +0000 Subject: [PATCH 17/25] fsdiff: fingerprint oversized files so unchanged ones don't diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files over MaxFileBytes were always reported as "Binary files … differ", so git-free runs, copy isolation and the stray-write guard saw phantom changes in folders holding large files. Withheld files now carry a streamed SHA-256; matching fingerprints are treated as unchanged. Co-Authored-By: Claude Opus 5.5 --- internal/fsdiff/diff.go | 2 ++ internal/fsdiff/fsdiff_test.go | 34 ++++++++++++++++++++++++++++++++++ internal/fsdiff/snapshot.go | 27 +++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/internal/fsdiff/diff.go b/internal/fsdiff/diff.go index 49b4612..ad46cc1 100644 --- a/internal/fsdiff/diff.go +++ b/internal/fsdiff/diff.go @@ -43,6 +43,8 @@ func Diff(dir string, before *Snapshot) (string, error) { writeRemoved(&b, p, o) case !hadO && hadC: writeAdded(&b, p, c) + case o.Withheld && c.Withheld && o.Sum == c.Sum && o.Sum != [32]byte{}: + // unchanged oversized file (fingerprints match) case o.Withheld || c.Withheld: writeOpaque(&b, p) // content unknown on at least one side case bytes.Equal(o.Content, c.Content): diff --git a/internal/fsdiff/fsdiff_test.go b/internal/fsdiff/fsdiff_test.go index 91d136b..f3e64a7 100644 --- a/internal/fsdiff/fsdiff_test.go +++ b/internal/fsdiff/fsdiff_test.go @@ -277,3 +277,37 @@ func TestLineEditsReconstructB(t *testing.T) { } } } + +func TestDiffOversizedFileUnchangedIsEmpty(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "big.log"), strings.Repeat("a", MaxFileBytes+1)) + before, err := Capture(dir) + if err != nil { + t.Fatal(err) + } + out, err := Diff(dir, before) + if err != nil { + t.Fatal(err) + } + if out != "" { + t.Fatalf("unchanged oversized file must not diff, got:\n%s", out) + } +} + +func TestDiffOversizedFileChangedIsOpaque(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "big.log") + writeFile(t, path, strings.Repeat("a", MaxFileBytes+1)) + before, err := Capture(dir) + if err != nil { + t.Fatal(err) + } + writeFile(t, path, strings.Repeat("a", MaxFileBytes)+"b") + out, err := Diff(dir, before) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "Binary files a/big.log and b/big.log differ") { + t.Fatalf("changed oversized file should diff as opaque, got:\n%s", out) + } +} diff --git a/internal/fsdiff/snapshot.go b/internal/fsdiff/snapshot.go index 47a61f1..0e240f2 100644 --- a/internal/fsdiff/snapshot.go +++ b/internal/fsdiff/snapshot.go @@ -12,7 +12,9 @@ package fsdiff import ( "bytes" + "crypto/sha256" "errors" + "io" "io/fs" "os" "path/filepath" @@ -36,7 +38,8 @@ var ErrTooLarge = errors.New("directory too large to snapshot (>128 MiB of file type File struct { Mode os.FileMode Content []byte - Withheld bool // too large to capture; treated as opaque + Withheld bool // too large to capture; treated as opaque + Sum [32]byte // SHA-256 of a withheld file, so unchanged ones don't diff } // Snapshot is the captured state of a directory. Keys are slash-separated, @@ -135,7 +138,11 @@ func Capture(dir string) (*Snapshot, error) { return nil } if info.Size() > MaxFileBytes { - snap.Files[rel] = &File{Mode: info.Mode().Perm(), Withheld: true} + f := &File{Mode: info.Mode().Perm(), Withheld: true} + if sum, herr := hashFile(p); herr == nil { + f.Sum = sum + } + snap.Files[rel] = f return nil } data, rerr := os.ReadFile(p) @@ -164,3 +171,19 @@ func looksBinary(data []byte) bool { } return bytes.IndexByte(data[:n], 0) >= 0 } + +// hashFile streams a file through SHA-256 without holding it in memory. +func hashFile(path string) ([32]byte, error) { + var sum [32]byte + f, err := os.Open(path) + if err != nil { + return sum, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return sum, err + } + copy(sum[:], h.Sum(nil)) + return sum, nil +} From 6ecf849816c1f0cfe5e5d86d412334c4018fdac1 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:06:12 +0000 Subject: [PATCH 18/25] history/benchmark/tui: show tokens and cost orchestra history and the benchmark leaderboard gain TOKENS/COST columns (and benchmarks persist usage). The dashboard shows them in History and Benchmarks, per-agent totals in Agents, and a running total in the live run panel; usage columns appear only when some row has usage, and narrow windows shed low-priority columns (new col.drop) instead of overflowing. Chat turns now record their summed usage to history. Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/benchmark.go | 11 +++- cmd/orchestra/history.go | 20 +++++- internal/tui/chat.go | 17 +++-- internal/tui/live.go | 7 +++ internal/tui/table.go | 35 ++++++++--- internal/tui/tui.go | 3 + internal/tui/usage_test.go | 123 +++++++++++++++++++++++++++++++++++++ internal/tui/views.go | 74 ++++++++++++++++++---- 8 files changed, 260 insertions(+), 30 deletions(-) create mode 100644 internal/tui/usage_test.go diff --git a/cmd/orchestra/benchmark.go b/cmd/orchestra/benchmark.go index 3f11e53..9e2c85f 100644 --- a/cmd/orchestra/benchmark.go +++ b/cmd/orchestra/benchmark.go @@ -327,6 +327,7 @@ func persistComparison(task, level string, pairs []abPair, agents []agent.Agent) Valid: r.valid(), Skipped: r.skipped(), Changed: r.out.HadChanges, Duration: r.dur, Retries: retriesOf(r), Files: r.files, Added: r.added, Removed: r.removed, Exit: r.out.ExitCode, Won: false, + TokensIn: r.out.Usage.InputTokens, TokensOut: r.out.Usage.OutputTokens, CostUSD: r.out.Usage.CostUSD, }, now) } for i := range agents { @@ -417,7 +418,7 @@ func firstMergeable(ranked []benchResult) (benchResult, bool) { func printLeaderboard(ranked []benchResult) { fmt.Printf("\n%s\n", ui.Heading("leaderboard")) - fmt.Printf("%-4s %-12s %-8s %-8s %-8s %-6s %s\n", "RANK", "AGENT", "VALID", "TIME", "RETRIES", "FILES", "±LINES") + fmt.Printf("%-4s %-12s %-8s %-8s %-8s %-6s %-11s %-7s %s\n", "RANK", "AGENT", "VALID", "TIME", "RETRIES", "FILES", "TOKENS", "COST", "±LINES") for i, r := range ranked { rank := fmt.Sprintf("%d", i+1) if i == 0 { @@ -444,9 +445,12 @@ func printLeaderboard(ranked []benchResult) { } else if !r.out.HadChanges { detail = " " + ui.Dim("(no changes)") } - fmt.Printf("%-4s %-12s %-8s %-8s %-8d %-6d %s%s\n", + u := r.out.Usage + fmt.Printf("%-4s %-12s %-8s %-8s %-8d %-6d %s %s %s%s\n", rank, r.agent, valid, - r.dur.Round(time.Second/10).String(), retriesOf(r), r.files, lines, detail) + r.dur.Round(time.Second/10).String(), retriesOf(r), r.files, + padCell(agent.TokensCell(u.InputTokens, u.OutputTokens), 11), padCell(agent.CostCell(u.CostUSD), 7), + lines, detail) } } @@ -465,6 +469,7 @@ func persistBenchmarks(_ context.Context, task string, ranked []benchResult) { Duration: r.dur, Retries: retriesOf(r), Files: r.files, Added: r.added, Removed: r.removed, Exit: r.out.ExitCode, Won: i == 0, + TokensIn: r.out.Usage.InputTokens, TokensOut: r.out.Usage.OutputTokens, CostUSD: r.out.Usage.CostUSD, }, now) } } diff --git a/cmd/orchestra/history.go b/cmd/orchestra/history.go index fa51259..95ade05 100644 --- a/cmd/orchestra/history.go +++ b/cmd/orchestra/history.go @@ -2,6 +2,10 @@ package main import ( "fmt" + "strings" + "unicode/utf8" + + "github.com/crossben/orchestra-code/internal/agent" "github.com/spf13/cobra" ) @@ -46,14 +50,15 @@ func newHistoryCmd() *cobra.Command { } } - fmt.Printf("%-19s %-10s %-10s %-4s %s\n", "WHEN", "AGENT", "OUTCOME", "ATT", "TASK") + fmt.Printf("%-19s %-10s %-10s %-4s %-11s %-7s %s\n", "WHEN", "AGENT", "OUTCOME", "ATT", "TOKENS", "COST", "TASK") for _, r := range runs { task := firstLine(r.Prompt) if len(task) > 48 { task = task[:47] + "…" } - fmt.Printf("%-19s %-10s %-10s %-4d %s\n", - r.Time.Local().Format("2006-01-02 15:04:05"), r.Agent, r.Outcome, r.Attempts, task) + fmt.Printf("%-19s %-10s %-10s %-4d %s %s %s\n", + r.Time.Local().Format("2006-01-02 15:04:05"), r.Agent, r.Outcome, r.Attempts, + padCell(agent.TokensCell(r.TokensIn, r.TokensOut), 11), padCell(agent.CostCell(r.CostUSD), 7), task) } return nil }, @@ -63,6 +68,15 @@ func newHistoryCmd() *cobra.Command { return cmd } +// padCell left-aligns s in w columns, counting runes (fmt's %-Ns counts bytes, +// which misaligns the multi-byte "—"). +func padCell(s string, w int) string { + if n := utf8.RuneCountInString(s); n < w { + return s + strings.Repeat(" ", w-n) + } + return s +} + func firstLine(s string) string { for i, r := range s { if r == '\n' { diff --git a/internal/tui/chat.go b/internal/tui/chat.go index 7aa9a50..3faa2c6 100644 --- a/internal/tui/chat.go +++ b/internal/tui/chat.go @@ -7,6 +7,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/crossben/orchestra-code/internal/agent" "github.com/crossben/orchestra-code/internal/engine" "github.com/crossben/orchestra-code/internal/memory" "github.com/crossben/orchestra-code/internal/ui" @@ -138,7 +139,7 @@ func (m Model) onTurn(msg turnMsg) (tea.Model, tea.Cmd) { } else { text = "↺ cancelled — any changes were reverted" } - m.record(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "cancelled", Attempts: t.Attempts}) + m.record(usageRun(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "cancelled", Attempts: t.Attempts}, t.Usage)) } m.messages = append(m.messages, chatLine{role: "sys", text: text}) m.logEvent("turn", "run cancelled") @@ -147,7 +148,7 @@ func (m Model) onTurn(msg turnMsg) (tea.Model, tea.Cmd) { m.messages = append(m.messages, chatLine{role: "sys", text: "✗ " + t.Err.Error()}) m.logEvent("error", t.Err.Error()) if msg.started { - m.record(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "failed", Attempts: t.Attempts}) + m.record(usageRun(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "failed", Attempts: t.Attempts}, t.Usage)) } case msg.answered: run.result = "answered" @@ -162,7 +163,7 @@ func (m Model) onTurn(msg turnMsg) (tea.Model, tea.Cmd) { resp = "(the agent made no file changes)" } m.messages = append(m.messages, chatLine{role: "agent", text: resp, agent: agentName}) - m.record(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "no-change", Attempts: t.Attempts, Passed: t.Report.Passed()}) + m.record(usageRun(memory.Run{Agent: agentName, Prompt: run.task, Outcome: "no-change", Attempts: t.Attempts, Passed: t.Report.Passed()}, t.Usage)) m.logEvent("turn", fmt.Sprintf("%s replied without changes", agentName)) ui.Notify("Orchestra", "Agent finished — "+firstLine(resp)) default: @@ -214,10 +215,10 @@ func (m Model) reject() (tea.Model, tea.Cmd) { func (m Model) finishReview(outcome string) Model { t := m.pending - m.record(memory.Run{ + m.record(usageRun(memory.Run{ Agent: m.pendAg, Prompt: m.pendTask, Outcome: outcome, Attempts: t.Attempts, Passed: t.Report.Passed(), Diff: t.Diff, - }) + }, t.Usage)) m.logEvent("turn", outcome) m.cstate = chatIdle m.pending = engine.Turn{} @@ -229,6 +230,12 @@ func (m Model) finishReview(outcome string) Model { return m } +// usageRun stamps a run's summed token usage onto its history record. +func usageRun(r memory.Run, u agent.Usage) memory.Run { + r.TokensIn, r.TokensOut, r.CostUSD = u.InputTokens, u.OutputTokens, u.CostUSD + return r +} + // --- rendering --- func (m Model) chatView() string { diff --git a/internal/tui/live.go b/internal/tui/live.go index 7f83e30..1ea8f7f 100644 --- a/internal/tui/live.go +++ b/internal/tui/live.go @@ -11,6 +11,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/charmbracelet/x/ansi" + "github.com/crossben/orchestra-code/internal/agent" "github.com/crossben/orchestra-code/internal/engine" "github.com/crossben/orchestra-code/internal/gitutil" ) @@ -47,6 +48,7 @@ type liveRun struct { agentDone bool exitCode int agentDur time.Duration + usage agent.Usage // summed over finished attempts (unknown for CLI agents) stages []stageState past []string // one summary line per finished attempt lines []string // agent output tail @@ -263,6 +265,7 @@ func (m *Model) applyEvent(e engine.Event) { case engine.EventAgentDone: run.agentDone = true run.exitCode, run.agentDur = e.ExitCode, e.Duration + run.usage = run.usage.Add(e.Usage) case engine.EventStageStart, engine.EventStageDone: state := stRunning if e.Kind == engine.EventStageDone { @@ -369,6 +372,10 @@ func (m Model) runPanel(w, h int) string { } } + if u := run.usage.String(); u != "" { + head = append(head, dimSty.Render("∑ "+u)) + } + // Output tail fills the rest. outH := h - 2 - len(head) - 2 // border, head, gap + label var body []string diff --git a/internal/tui/table.go b/internal/tui/table.go index 1d33293..4cf7bcb 100644 --- a/internal/tui/table.go +++ b/internal/tui/table.go @@ -8,9 +8,12 @@ import ( ) // col is one table column: a fixed width (0 = take the remaining space). +// drop > 0 marks a column that may be hidden when the window is too narrow; +// the highest drop goes first. type col struct { title string width int + drop int } // fit pads or truncates s to exactly w terminal cells. Widths are measured on @@ -27,12 +30,16 @@ func fit(s string, w int) string { } // table lays out rows of pre-styled cells under a header, sized to width. -// The one zero-width column absorbs whatever space the others leave. +// The one zero-width column absorbs whatever space the others leave; +// droppable columns are hidden, highest drop first, until it keeps minFlex. type table struct { cols []col width int } +const minFlex = 12 + +// widths returns each column's width; a hidden column gets -1. func (t table) widths() []int { ws := make([]int, len(t.cols)) used, flex := 0, -1 @@ -43,6 +50,19 @@ func (t table) widths() []int { } used += c.width + 1 } + for used+minFlex > t.width { + worst := -1 + for i, c := range t.cols { + if ws[i] >= 0 && c.drop > 0 && (worst < 0 || c.drop > t.cols[worst].drop) { + worst = i + } + } + if worst < 0 { + break + } + used -= t.cols[worst].width + 1 + ws[worst] = -1 + } if flex >= 0 { ws[flex] = max(t.width-used, 8) } @@ -50,20 +70,19 @@ func (t table) widths() []int { } func (t table) header() string { - ws := t.widths() - cells := make([]string, len(t.cols)) + titles := make([]string, len(t.cols)) for i, c := range t.cols { - cells[i] = fit(c.title, ws[i]) + titles[i] = c.title } - return headSty.Render(strings.Join(cells, " ")) + return headSty.Render(t.row(titles...)) } func (t table) row(cells ...string) string { ws := t.widths() - out := make([]string, len(cells)) + out := make([]string, 0, len(cells)) for i, c := range cells { - if i < len(ws) { - out[i] = fit(c, ws[i]) + if i < len(ws) && ws[i] >= 0 { + out = append(out, fit(c, ws[i])) } } return strings.Join(out, " ") diff --git a/internal/tui/tui.go b/internal/tui/tui.go index a067d9d..741618f 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -216,6 +216,9 @@ func (m *Model) record(r memory.Run) { st.Accepted++ } st.LastUsed = r.Time + st.TokensIn += r.TokensIn + st.TokensOut += r.TokensOut + st.CostUSD += r.CostUSD m.stats[r.Agent] = st } m.reload() diff --git a/internal/tui/usage_test.go b/internal/tui/usage_test.go new file mode 100644 index 0000000..5b5a0d0 --- /dev/null +++ b/internal/tui/usage_test.go @@ -0,0 +1,123 @@ +package tui + +import ( + "strings" + "testing" + "time" + + "github.com/charmbracelet/lipgloss" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/memory" +) + +// usageModel is a model with history, benchmarks and agent stats where the +// api agent reported usage and the cli agent did not. +func usageModel(w, h int) Model { + m := testModelIn(".", nil, nil, w, h) + m.runs = []memory.Run{ + {Agent: "alpha", Prompt: "api run", Outcome: "accepted", Attempts: 2, TokensIn: 12_345, TokensOut: 1_100, CostUSD: 0.042}, + {Agent: "beta", Prompt: "cli run", Outcome: "rejected", Attempts: 1}, + } + m.benches = []memory.BenchRow{ + {Agent: "alpha", Task: "race", Valid: true, Won: true, Duration: time.Second, TokensIn: 4_000, TokensOut: 300, CostUSD: 0.02}, + {Agent: "beta", Task: "race", Duration: 2 * time.Second}, + } + m.stats = map[string]memory.AgentStats{ + "alpha": {Runs: 2, Accepted: 1, LastUsed: time.Now(), TokensIn: 15_000, TokensOut: 1_000, CostUSD: 0.06}, + "beta": {Runs: 1, LastUsed: time.Now()}, + } + m.reload() + return m +} + +func mustShow(t *testing.T, view string, want ...string) { + t.Helper() + for _, w := range want { + if !strings.Contains(view, w) { + t.Fatalf("view missing %q\n---\n%s", w, view) + } + } +} + +func TestUsageColumnsRender(t *testing.T) { + m := usageModel(160, 30) + mustShow(t, m.switchTab(tabHistory).View(), "TOKENS", "COST", "12.3k/1.1k", "$0.04", "—") + mustShow(t, m.switchTab(tabBench).View(), "TOKENS", "COST", "4.0k/300", "$0.02") + mustShow(t, m.switchTab(tabAgents).View(), "TOKENS", "COST", "15.0k/1.0k", "$0.06") +} + +// With only CLI agents (no usage anywhere) the columns stay out of the way. +func TestUsageColumnsHiddenWithoutUsage(t *testing.T) { + m := testModelIn(".", nil, nil, 160, 30) + m.runs = []memory.Run{{Agent: "beta", Prompt: "cli run", Outcome: "accepted", Attempts: 1}} + m.benches = []memory.BenchRow{{Agent: "beta", Task: "race"}} + m.reload() + for _, tb := range []tab{tabHistory, tabBench, tabAgents} { + if v := m.switchTab(tb).View(); strings.Contains(v, "TOKENS") { + t.Fatalf("tab %s should hide usage columns without usage:\n%s", tabNames[tb], v) + } + } +} + +// On narrow terminals low-priority columns give way; tokens stay visible in +// History and Benchmarks and nothing overflows. +func TestUsageColumnsFitNarrowWindows(t *testing.T) { + for _, size := range [][2]int{{80, 24}, {100, 30}, {140, 40}} { + m := usageModel(size[0], size[1]) + for _, tb := range []tab{tabHistory, tabBench, tabAgents} { + v := m.switchTab(tb).View() + for i, l := range strings.Split(v, "\n") { + if w := lipgloss.Width(l); w > size[0] { + t.Fatalf("%dx%d tab %s line %d is %d wide:\n%s", size[0], size[1], tabNames[tb], i, w, l) + } + } + if tb != tabAgents { + mustShow(t, v, "TOKENS") + } + } + } +} + +func TestLiveRunPanelShowsUsage(t *testing.T) { + m := testModelIn(".", nil, nil, 140, 40) + m.cstate = chatRunning + m.run = &liveRun{start: time.Now(), agent: "alpha"} + u := agent.Usage{InputTokens: 1000, OutputTokens: 100, CostUSD: 0.0045, Known: true, Priced: true} + for _, e := range []engine.Event{ + {Kind: engine.EventAttempt, Attempt: 1, Max: 2}, + {Kind: engine.EventAgentDone, Attempt: 1, Max: 2, Usage: u}, + {Kind: engine.EventAttempt, Attempt: 2, Max: 2}, + {Kind: engine.EventAgentDone, Attempt: 2, Max: 2, Usage: u}, + } { + m.applyEvent(e) + } + mustShow(t, m.View(), "tokens 2.0k in / 200 out · $0.0090") +} + +// A reviewed turn's summed usage lands in History and the agent's totals. +func TestFinishReviewRecordsUsage(t *testing.T) { + m := testModel() + m.pending = engine.Turn{Attempts: 2, Usage: agent.Usage{InputTokens: 2000, OutputTokens: 200, CostUSD: 0.009, Known: true, Priced: true}} + m.pendAg, m.pendTask = "alpha", "task" + m = m.finishReview("accepted") + if len(m.runs) != 1 || m.runs[0].TokensIn != 2000 || m.runs[0].TokensOut != 200 || m.runs[0].CostUSD != 0.009 { + t.Fatalf("run not recorded with usage: %+v", m.runs) + } + if st := m.stats["alpha"]; st.TokensIn != 2000 || st.CostUSD != 0.009 { + t.Fatalf("agent totals not updated: %+v", st) + } + mustShow(t, m.switchTab(tabHistory).View(), "2.0k/200", "$0.0090") +} + +func TestLiveRunPanelHidesUnknownUsage(t *testing.T) { + m := testModelIn(".", nil, nil, 140, 40) + m.cstate = chatRunning + m.run = &liveRun{start: time.Now(), agent: "alpha"} + m.applyEvent(engine.Event{Kind: engine.EventAttempt, Attempt: 1, Max: 1}) + m.applyEvent(engine.Event{Kind: engine.EventAgentDone, Attempt: 1, Max: 1}) + if strings.Contains(m.View(), "tokens") { + t.Fatalf("unknown usage must not render:\n%s", m.View()) + } +} diff --git a/internal/tui/views.go b/internal/tui/views.go index 80e3cd7..ea41fda 100644 --- a/internal/tui/views.go +++ b/internal/tui/views.go @@ -49,7 +49,7 @@ func (m Model) changesView() string { if len(idx) == 0 { return emptyState("No changes yet", "Accepted and rejected diffs from Chat show up here — and stay after a restart.", m.width, h) } - t := table{width: m.width - 2, cols: []col{{"WHEN", 12}, {"AGENT", 12}, {"OUTCOME", 12}, {"FILES", 5}, {"LINES", 12}, {"TASK", 0}}} + t := table{width: m.width - 2, cols: []col{{"WHEN", 12, 0}, {"AGENT", 12, 0}, {"OUTCOME", 12, 0}, {"FILES", 5, 0}, {"LINES", 12, 0}, {"TASK", 0, 0}}} var b strings.Builder b.WriteString(" " + t.header() + "\n") start, end := listWindow(len(idx), m.changeSel, h-1) @@ -68,7 +68,17 @@ func (m Model) historyView() string { if len(m.runs) == 0 { return emptyState("No run history yet", "Runs from Chat, orchestra run and orchestra do are listed here.", m.width, h) } - t := table{width: m.width - 2, cols: []col{{"WHEN", 12}, {"AGENT", 12}, {"OUTCOME", 12}, {"TRIES", 5}, {"CHECKS", 7}, {"LINES", 12}, {"TASK", 0}}} + usage := false + for _, r := range m.runs { + usage = usage || r.TokensIn+r.TokensOut > 0 + } + cols := []col{{"WHEN", 12, 0}, {"AGENT", 12, 0}, {"OUTCOME", 12, 0}, {"TRIES", 5, 0}, {"CHECKS", 7, 0}, {"LINES", 12, 0}} + if usage { + // Narrow windows shed CHECKS, then TRIES, then COST; TOKENS stays. + cols[3].drop, cols[4].drop = 2, 3 + cols = append(cols, col{"TOKENS", 10, 0}, col{"COST", 7, 1}) + } + t := table{width: m.width - 2, cols: append(cols, col{"TASK", 0, 0})} var b strings.Builder b.WriteString(" " + t.header() + "\n") start, end := listWindow(len(m.runs), m.histSel, h-1) @@ -82,15 +92,28 @@ func (m Model) historyView() string { if st.files > 0 { lines = counts(st.added, st.removed) } - b.WriteString(selRow(i == m.histSel, t.row( - dimSty.Render(whenCell(r.Time)), youSty.Render(r.Agent), outcomeCell(r.Outcome), - fmt.Sprint(r.Attempts), checks, lines, firstLine(r.Prompt))) + "\n") + cells := []string{dimSty.Render(whenCell(r.Time)), youSty.Render(r.Agent), outcomeCell(r.Outcome), + fmt.Sprint(r.Attempts), checks, lines} + if usage { + cells = append(cells, tokensCell(r.TokensIn, r.TokensOut), costCell(r.CostUSD)) + } + b.WriteString(selRow(i == m.histSel, t.row(append(cells, firstLine(r.Prompt))...)) + "\n") } return b.String() } func (m Model) agentsView() string { - t := table{width: m.width - 2, cols: []col{{"AGENT", 14}, {"INSTALLED", 10}, {"LIVE PROBE", 26}, {"RUNS", 5}, {"ACCEPTED", 9}, {"LAST USED", 13}, {"CAN", 0}}} + usage := false + for _, st := range m.stats { + usage = usage || st.TokensIn+st.TokensOut > 0 + } + cols := []col{{"AGENT", 14, 0}, {"INSTALLED", 10, 0}, {"LIVE PROBE", 26, 0}, {"RUNS", 5, 0}, {"ACCEPTED", 9, 0}, {"LAST USED", 13, 0}} + if usage { + // Per-agent totals; narrow windows shed LAST USED, then COST, then TOKENS. + cols[5].drop = 3 + cols = append(cols, col{"TOKENS", 10, 1}, col{"COST", 7, 2}) + } + t := table{width: m.width - 2, cols: append(cols, col{"CAN", 0, 0})} var b strings.Builder b.WriteString(" " + t.header() + "\n") for _, a := range m.d.Reg.All() { @@ -120,7 +143,11 @@ func (m Model) agentsView() string { rate = fmt.Sprintf("%d%%", st.Accepted*100/st.Runs) last = dimSty.Render(whenCell(st.LastUsed)) } - b.WriteString(" " + t.row(name, installed, probe, runs, rate, last, dimSty.Render(caps(a))) + "\n") + cells := []string{name, installed, probe, runs, rate, last} + if usage { + cells = append(cells, tokensCell(st.TokensIn, st.TokensOut), costCell(st.CostUSD)) + } + b.WriteString(" " + t.row(append(cells, dimSty.Render(caps(a)))...) + "\n") } b.WriteString("\n " + dimSty.Render("★ default agent · ACCEPTED = share of runs whose changes you kept · p probes whether each agent can actually run")) return b.String() @@ -131,7 +158,17 @@ func (m Model) benchView() string { if len(m.benches) == 0 { return emptyState("No benchmarks yet", "Run orchestra benchmark \"\" to race every agent on one task.", m.width, h) } - t := table{width: m.width - 2, cols: []col{{"WHEN", 12}, {"AGENT", 12}, {"WON", 4}, {"VALID", 6}, {"TIME", 8}, {"RETRIES", 8}, {"LINES", 12}, {"TASK", 0}}} + usage := false + for _, r := range m.benches { + usage = usage || r.TokensIn+r.TokensOut > 0 + } + cols := []col{{"WHEN", 12, 0}, {"AGENT", 12, 0}, {"WON", 4, 0}, {"VALID", 6, 0}, {"TIME", 8, 0}, {"RETRIES", 8, 0}, {"LINES", 12, 0}} + if usage { + // Narrow windows shed RETRIES, then COST, then TIME; TOKENS stays. + cols[4].drop, cols[5].drop = 1, 3 + cols = append(cols, col{"TOKENS", 10, 0}, col{"COST", 7, 2}) + } + t := table{width: m.width - 2, cols: append(cols, col{"TASK", 0, 0})} var b strings.Builder b.WriteString(" " + t.header() + "\n") for i, r := range m.benches { @@ -146,8 +183,12 @@ func (m Model) benchView() string { if r.Valid { valid = okSty.Render("✓") } - b.WriteString(" " + t.row(dimSty.Render(whenCell(r.Time)), youSty.Render(r.Agent), won, valid, - r.Duration.Round(time.Second/10).String(), fmt.Sprint(r.Retries), counts(r.Added, r.Removed), firstLine(r.Task)) + "\n") + cells := []string{dimSty.Render(whenCell(r.Time)), youSty.Render(r.Agent), won, valid, + r.Duration.Round(time.Second / 10).String(), fmt.Sprint(r.Retries), counts(r.Added, r.Removed)} + if usage { + cells = append(cells, tokensCell(r.TokensIn, r.TokensOut), costCell(r.CostUSD)) + } + b.WriteString(" " + t.row(append(cells, firstLine(r.Task))...) + "\n") } return b.String() } @@ -157,7 +198,7 @@ func (m Model) logsView() string { if len(m.logs) == 0 { return emptyState("Nothing logged yet", "Routing, runs, retries, probes and errors from this session appear here.", m.width, h) } - t := table{width: m.width - 2, cols: []col{{"TIME", 9}, {"KIND", 6}, {"MESSAGE", 0}}} + t := table{width: m.width - 2, cols: []col{{"TIME", 9, 0}, {"KIND", 6, 0}, {"MESSAGE", 0, 0}}} var b strings.Builder b.WriteString(" " + t.header() + "\n") start := max(len(m.logs)-(h-1), 0) @@ -178,6 +219,17 @@ func (m Model) logsView() string { return b.String() } +// tokensCell / costCell render stored usage compactly, dimming "—" (unknown). +func tokensCell(in, out int) string { return dimDash(agent.TokensCell(in, out)) } +func costCell(usd float64) string { return dimDash(agent.CostCell(usd)) } + +func dimDash(s string) string { + if s == "—" { + return dimSty.Render(s) + } + return s +} + func caps(a agent.Agent) string { out := make([]string, 0, len(a.Capabilities())) for _, c := range a.Capabilities() { From dd6d4c3a9f6dcd9de27ceb3c4ea1c446653ac1ee Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:06:12 +0000 Subject: [PATCH 19/25] docs: document API-agent token tracking and price fields Co-Authored-By: Claude Opus 5.5 --- docs/EXTENDING.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 4507d05..dd6d7c2 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -92,6 +92,30 @@ Notes: - Works everywhere CLI agents do: `run`, shell, dashboard chat, planning, routing, and even `do --parallel` worktrees (patches apply inside each isolated worktree). +### Tokens and cost + +API agents record the token usage the provider reports (OpenAI-compatible +`usage.prompt_tokens`/`completion_tokens`, Anthropic `usage.input_tokens`/`output_tokens`), +summed over every attempt of a run including self-correction retries. Gateways that omit +usage simply record nothing. To also see cost, set the model's prices — Orchestra ships no +price table, because prices change: + +```yaml +agents: + - name: sonnet + type: api + provider: anthropic + model: claude-sonnet-4-5 + price_input_per_mtok: 3 # USD per 1M input tokens + price_output_per_mtok: 15 # USD per 1M output tokens +``` + +Cost is computed only when both prices are set (> 0); otherwise tokens are still recorded +and cost shows as `—`. Usage appears in the end-of-run line +(`▸ tokens 12.3k in / 1.1k out · $0.04`), `orchestra history`, the `benchmark` leaderboard, +and the dashboard's History, Benchmarks and Agents (per-agent totals) tabs. CLI agents +don't report usage yet, so their runs show `—`. + --- ## 3. Implement the `Agent` interface (for a built-in / non-CLI agent) From 9406dbc3c77c240c0570402452983769830a11d3 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:07:56 +0000 Subject: [PATCH 20/25] docs: AGENTS.md and plan.md for router v2, git-free parallel, usage, updater Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 18 +++++++++++------- plan.md | 9 ++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ed2f59c..b1dacbd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,13 +37,14 @@ Binary is at `bin/orchestra` (gitignored). - `cmd/orchestra/` — Cobra CLI (main.go + one file per subcommand: run, do, plan, dashboard, agents, etc.) - `internal/engine/` — the core supervised loop (dispatch → validate → retry → review). Every front door (`run`, shell, `do`) drives this. - `internal/agent/` — Agent interface + registry. This is the extension point for adding new agents. -- `internal/router/` — AI routing: classifies intent → picks agent. Uses a CLI agent now; API planned. +- `internal/router/` — AI routing: classifies intent → picks agent. CLI classifier by default; optional direct-API classifier (`router.classifier`); history-aware agent resolution via the `History` interface. - `internal/planner/` — decomposes requests into ordered steps with `depends_on` for parallelism. -- `internal/scheduler/` — bounded-concurrency DAG runner for parallel worktrees. -- `internal/worktree/` — git worktree isolation (branch per task, merge + conflict detection). +- `internal/scheduler/` — bounded-concurrency DAG runner for parallel isolated tasks. +- `internal/worktree/` — task isolation behind the `Isolator` interface: `worktree.New(dir)` picks git worktrees in a repository (branch per task, merge + conflict detection) or a `CopyIsolator` in plain folders (temp copy per task, all-or-nothing merge, conflict if a touched file changed in the source). - `internal/validate/` — build → lint → test pipeline, auto-detects toolchain, stops on first failure. - `internal/config/` — YAML config + built-in agent defaults + toolchain auto-detection. -- `internal/memory/` — SQLite history at `~/.orchestra/orchestra.db` (never in the repo tree). +- `internal/memory/` — SQLite history at `~/.orchestra/orchestra.db` (never in the repo tree): runs, diffs, benchmarks, tokens/cost, and the per-directory agent scores the router uses. +- `internal/update/` — self-updater: checks GitHub Releases (throttled to once per 24h), verifies `checksums.txt`, swaps the binary in place. Relies on the GoReleaser archive name template — see docs/RELEASING.md. - `internal/tui/` — Bubble Tea dashboard: Chat (live run panel + file-by-file review), Changes, History, Agents, Benchmarks, Logs. One file per concern (chat, live, review, views, chrome, theme, table). - `internal/ui/` — terminal styling (gradients, spinners, diffs). TTY-aware. - `internal/shell/` — interactive chat REPL. @@ -75,7 +76,8 @@ dispatch → validate → retry (self-correct) → review (accept/reject) Git is optional. Inside a repository, diffs and rejects use git and the tree must start clean (reject does `git restore`). Outside one — like opencode or claude — the engine snapshots the directory first (`internal/fsdiff`), diffs against that snapshot, and restores from it on reject; -accept just keeps the files. `--parallel` and `benchmark` still require a repository (worktrees). +accept just keeps the files. `--parallel` and `benchmark` work in both: worktrees inside a repository, +per-task directory copies (`worktree.CopyIsolator`) outside one. If you change validation or retry logic, you change all three entry points. The engine is the product — not the CLI or the TUI. @@ -124,9 +126,11 @@ Live runs: the dashboard calls `engine.Produce` with `OnEvent` (attempt / agent- These are not interchangeable with other TUI frameworks. Tests are headless (`tui_test.go` renders views without a TTY by injecting `WindowSizeMsg`); end-to-end chat tests drive the real worker with fake-agent shell scripts. -## Router is CLI-based +## Router -The AI router (`internal/router/`) classifies intent by dispatching to an agent in query mode (e.g. `claude -p`), not by calling an LLM API itself. It inherits agent CLI quirks: needs `--dangerously-skip-permissions`, uses the same `runner` package, and the classification latency is whatever the agent takes. Since `APIAgent` also implements `Querier`, an api-type agent can serve as the router's classifier today; a dedicated direct-API classifier inside `internal/router` remains planned but unimplemented. +By default the AI router (`internal/router/`) classifies intent by dispatching to an agent in query mode (e.g. `claude -p`). That inherits agent CLI quirks: needs `--dangerously-skip-permissions`, uses the same `runner` package, and the classification latency is whatever the agent takes. Setting `router.classifier: {provider, model}` switches classification to `APIClassifier`, one direct `llm.Provider` call (falls back to the CLI classifier if the key is missing); `router.agent` still answers questions. + +Agent resolution: AI suggestion → history pick (`History` interface, implemented by `memory.Store`: Laplace-smoothed accept rate over runs + benchmarks in this directory, ≥3 samples, strict winner) → static `router.routes` → default agent → first healthy. The router never imports memory. ## Gitignored directories diff --git a/plan.md b/plan.md index 262df91..d97506a 100644 --- a/plan.md +++ b/plan.md @@ -248,7 +248,8 @@ The nice-to-haves that make the project stand out — build only once the core i - **Benchmark mode ✅ DONE:** `orchestra benchmark ""` runs the task through every available agent, each in an isolated git worktree (parallel), then ranks a leaderboard by validation → retries → speed → diff size, offers to merge the winner, and records results to SQLite (`benchmarks` table) - for future data-driven routing. Tokens/cost deferred (agent CLIs don't report usage generically). + for data-driven routing. **Tokens/cost ✅ (API agents):** usage is recorded per run/benchmark and priced + from optional `price_input_per_mtok`/`price_output_per_mtok`; CLI agents stay "—" (no generic usage report). - **Plugin SDK ✅ DONE (lightweight):** the `Agent` interface is the extension point. Any CLI agent is added via `orchestra.yaml` (name/bin/args/dir_flag/capabilities) with **zero code**; in-process agents implement the interface (+ optional Querier/Prober/QuietRunner/QuietQuerier). Documented in @@ -261,6 +262,12 @@ The nice-to-haves that make the project stand out — build only once the core i - **Git-free supervision ✅ DONE:** outside a repository the engine snapshots the directory (`internal/fsdiff`), shows a git-style diff against the snapshot, and restores it on reject — so `run`/`do`/shell work in plain folders. `--parallel` and `benchmark` still require git (worktrees). +- **Router v2 ✅ DONE (2026-09):** optional direct-API classifier (`router.classifier`) and history-aware + agent resolution from accepted runs + benchmark wins (design: docs/superpowers/specs/2026-09-26-router-v2-design.md). +- **Git-free parallel & benchmark ✅ DONE (2026-09):** `worktree.Isolator` with a copy isolator for plain + folders (design: docs/superpowers/specs/2026-09-26-gitfree-parallel-design.md). +- **Self-updater ✅ DONE (2026-09):** launch-time check (24h throttle, TTY only), checksum-verified in-place + replace, exit to relaunch; `orchestra update` (design: docs/superpowers/specs/2026-09-26-auto-update-design.md). - **Context engine:** *deferred* — the supported agents gather their own repo context, so pre-selecting a file slice is largely redundant. Revisit only if token/cost becomes a measured problem. From f8940cb8da03579a6da0ccc30d994057ef8eb98a Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:08:53 +0000 Subject: [PATCH 21/25] parallel: extract shared wave logic from do --parallel New internal/parallel package holds the UI-free pieces of the parallel workflow: step id/prompt/agent helpers, a dependency Graph (ready waves, done/dead bookkeeping, waves-left estimate), the base-tree stray-write BaseGuard, and RunStep (quiet Produce + commit on a worktree branch) for UIs. do --parallel now uses it; its output and behaviour are unchanged. engine exports CommitMessage. Adds the dashboard-parallel design spec. Co-Authored-By: Claude Opus 5.5 --- cmd/orchestra/do.go | 146 +++---------- .../2026-09-26-dashboard-parallel-design.md | 82 ++++++++ internal/engine/engine.go | 4 + internal/parallel/parallel.go | 191 ++++++++++++++++++ internal/parallel/parallel_test.go | 179 ++++++++++++++++ 5 files changed, 487 insertions(+), 115 deletions(-) create mode 100644 docs/superpowers/specs/2026-09-26-dashboard-parallel-design.md create mode 100644 internal/parallel/parallel.go create mode 100644 internal/parallel/parallel_test.go diff --git a/cmd/orchestra/do.go b/cmd/orchestra/do.go index 29e2825..4742015 100644 --- a/cmd/orchestra/do.go +++ b/cmd/orchestra/do.go @@ -6,15 +6,14 @@ import ( "errors" "fmt" "os" - "strconv" "strings" "github.com/crossben/orchestra-code/internal/agent" "github.com/crossben/orchestra-code/internal/config" "github.com/crossben/orchestra-code/internal/engine" - "github.com/crossben/orchestra-code/internal/fsdiff" "github.com/crossben/orchestra-code/internal/gitutil" "github.com/crossben/orchestra-code/internal/memory" + "github.com/crossben/orchestra-code/internal/parallel" "github.com/crossben/orchestra-code/internal/planner" "github.com/crossben/orchestra-code/internal/review" "github.com/crossben/orchestra-code/internal/scheduler" @@ -28,7 +27,7 @@ func newDoCmd() *cobra.Command { var ( agentName string yes bool - parallel bool + inParallel bool jobs int principles string ) @@ -86,8 +85,8 @@ func newDoCmd() *cobra.Command { fmt.Printf("%s planning with %s\n", ui.Accent("▸"), ui.Agent(p.AgentName())) sp := ui.Spin("planning…") var pl planner.Plan - if parallel { - pl, err = p.MakeParallel(cmd.Context(), request, flagDir, healthyAgentNames(reg)) + if inParallel { + pl, err = p.MakeParallel(cmd.Context(), request, flagDir, parallel.HealthyAgentNames(reg)) } else { pl, err = p.Make(cmd.Context(), request, flagDir) } @@ -113,7 +112,7 @@ func newDoCmd() *cobra.Command { } stages := stagesFor(cfg) - if parallel { + if inParallel { return runParallel(cmd.Context(), in, cfg, reg, ag, agentName, pl, stages, jobs, mem) } return runSequential(cmd.Context(), in, cfg, reg, ag, agentName, pl, stages, mem) @@ -121,8 +120,8 @@ func newDoCmd() *cobra.Command { } cmd.Flags().StringVar(&agentName, "agent", "", "agent for planning and implementation (default from config)") cmd.Flags().BoolVar(&yes, "yes", false, "skip the plan-approval prompt") - cmd.Flags().BoolVar(¶llel, "parallel", false, "run independent steps concurrently in isolated worktrees (or folder copies)") - cmd.Flags().IntVar(&jobs, "jobs", 4, "max concurrent steps when --parallel") + cmd.Flags().BoolVar(&inParallel, "parallel", false, "run independent steps concurrently in isolated worktrees (or folder copies)") + cmd.Flags().IntVar(&jobs, "jobs", parallel.DefaultJobs, "max concurrent steps when --parallel") cmd.Flags().StringVar(&principles, "principles", "", "lean-code principles preamble: off|lite|full (default from config)") return cmd } @@ -132,13 +131,13 @@ func newDoCmd() *cobra.Command { func runSequential(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg *agent.Registry, ag agent.Agent, agentName string, pl planner.Plan, stages []validate.Stage, mem *memory.Store) error { retries := cfg.RetryLimit() for i, step := range pl.Steps { - stepAgent := stepAgentFor(reg, step, ag, agentName) + stepAgent := parallel.StepAgent(reg, step, ag, agentName) fmt.Printf("\n%s %s %s\n", ui.Accent(fmt.Sprintf("═══ step %d/%d", i+1, len(pl.Steps))), ui.Heading(step.Title), ui.Dim("["+stepAgent.Name()+"]")) out, err := engine.Execute(ctx, in, engine.Options{ Agent: stepAgent, - Prompt: stepTask(step), + Prompt: parallel.StepTask(step), Dir: flagDir, Stages: stages, MaxRetries: retries, @@ -170,24 +169,15 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg return err } defer mgr.Cleanup() - guard := newBaseGuard(flagDir) + guard := parallel.NewBaseGuard(flagDir) - nodes := make([]scheduler.Node, len(pl.Steps)) - for i, s := range pl.Steps { - deps := make([]string, 0, len(s.DependsOn)) - for _, d := range s.DependsOn { - deps = append(deps, strconv.Itoa(d)) - } - nodes[i] = scheduler.Node{ID: strconv.Itoa(i + 1), Deps: deps} - } - if err := scheduler.Validate(nodes); err != nil { + g, err := parallel.NewGraph(pl) + if err != nil { return err } retries := cfg.RetryLimit() - done := map[string]bool{} - dead := map[string]bool{} - id := func(i int) string { return strconv.Itoa(i + 1) } + id := parallel.StepID type result struct { tree worktree.Tree @@ -197,12 +187,12 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg } for wave := 1; ; wave++ { - ready := scheduler.Ready(nodes, done, dead) + ready := g.Ready() if len(ready) == 0 { break } fmt.Printf("\n%s\n", ui.Heading(fmt.Sprintf("── wave %d: %d step(s) in parallel ──", wave, len(ready)))) - if err := guard.arm(); err != nil { + if err := guard.Arm(); err != nil { return err } @@ -216,8 +206,8 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg return aerr } out, rerr := engine.ExecuteHeadless(ctx, engine.Options{ - Agent: stepAgentFor(reg, pl.Steps[i], ag, agentName), - Prompt: stepTask(pl.Steps[i]), + Agent: parallel.StepAgent(reg, pl.Steps[i], ag, agentName), + Prompt: parallel.StepTask(pl.Steps[i]), Dir: tree.Dir, Stages: stages, MaxRetries: retries, @@ -234,7 +224,13 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg // worktree (some CLIs don't honor the working directory), the base is now // dirty and merges would fail. Discard that stray work so the properly // isolated branches can still merge cleanly. - guard.check() + if guard.Check() { + if guard.InRepo() { + fmt.Println(ui.Warn(" ! an agent wrote outside its worktree — discarding stray changes in the base tree")) + } else { + fmt.Println(ui.Warn(" ! an agent wrote outside its isolated copy — discarding stray changes in the base folder")) + } + } // Fan-in: review + merge each result in order. for k, i := range ready { @@ -244,7 +240,7 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg if r.err != nil { fmt.Printf(" %s %v\n", ui.Danger("failed:"), r.err) - dead[id(i)] = true + g.MarkDead(i) if r.created { mgr.Remove(r.tree) } @@ -253,13 +249,13 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg if r.out.ExitCode != 0 { fmt.Printf(" %s agent exited abnormally (code %d) — skipping this step\n", ui.Danger("✗"), r.out.ExitCode) - dead[id(i)] = true + g.MarkDead(i) mgr.Remove(r.tree) continue } if !r.out.HadChanges { fmt.Println(ui.Dim(" no changes produced — nothing to merge")) - done[id(i)] = true + g.MarkDone(i) mgr.Remove(r.tree) continue } @@ -277,20 +273,20 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg } if conflict { fmt.Println(ui.Danger(" ✗ merge conflict — left unmerged; dependent steps will be skipped")) - dead[id(i)] = true + g.MarkDead(i) } else { fmt.Println(ui.Success(" ✓ merged into base")) - done[id(i)] = true + g.MarkDone(i) } } else { fmt.Println(ui.Warn(" ↺ rejected — discarded")) - dead[id(i)] = true + g.MarkDead(i) } mgr.Remove(r.tree) } } - merged := len(done) + merged := g.Merged() if merged == len(pl.Steps) { fmt.Printf("\n%s\n", ui.Success(fmt.Sprintf("✓ parallel workflow complete — all %d steps merged", merged))) } else { @@ -299,36 +295,6 @@ func runParallel(ctx context.Context, in *bufio.Reader, cfg *config.Config, reg return nil } -// stepAgentFor honors a planner-assigned per-step agent when valid+healthy, -// otherwise falls back to the workflow agent. -func stepAgentFor(reg *agent.Registry, step planner.Step, fallback agent.Agent, fallbackName string) agent.Agent { - if step.Agent != "" && step.Agent != fallbackName { - if a, ok := reg.Get(step.Agent); ok && a.Health() == nil { - return a - } - } - return fallback -} - -// healthyAgentNames returns the names of installed/available agents, offered to -// the planner as per-step agent choices. -func healthyAgentNames(reg *agent.Registry) []string { - var names []string - for _, a := range reg.All() { - if a.Health() == nil { - names = append(names, a.Name()) - } - } - return names -} - -func stepTask(step planner.Step) string { - if step.Detail == "" { - return step.Title - } - return step.Title + "\n" + step.Detail -} - // confirm reads a y/N answer from the shared reader (default no). func confirm(in *bufio.Reader, question string) bool { fmt.Printf("%s [y/N] ", ui.Accent(question)) @@ -344,53 +310,3 @@ func confirm(in *bufio.Reader, question string) bool { return false } } - -// baseGuard catches agents that wrote into the base directory instead of their -// isolated tree (some CLIs don't honor the working directory) and discards that -// stray work. Inside a repository "stray" means the tree is no longer clean; -// in a plain folder it means the folder differs from a snapshot armed before -// the wave's fan-out. -type baseGuard struct { - dir string - inRepo bool - snap *fsdiff.Snapshot -} - -func newBaseGuard(dir string) *baseGuard { - return &baseGuard{dir: dir, inRepo: gitutil.IsRepo(dir)} -} - -// arm records the base state before a wave runs (a no-op inside a repository, -// where the clean tree is the reference). -func (g *baseGuard) arm() error { - if g.inRepo { - return nil - } - snap, err := fsdiff.Capture(g.dir) - if err != nil { - return fmt.Errorf("snapshot %s: %w", g.dir, err) - } - g.snap = snap - return nil -} - -// check discards any change made to the base since arm, reporting whether it did. -func (g *baseGuard) check() bool { - if g.inRepo { - if clean, _ := gitutil.IsClean(g.dir); !clean { - fmt.Println(ui.Warn(" ! an agent wrote outside its worktree — discarding stray changes in the base tree")) - _ = gitutil.Restore(g.dir) - return true - } - return false - } - if g.snap == nil { - return false - } - if d, err := fsdiff.Diff(g.dir, g.snap); err != nil || d == "" { - return false - } - fmt.Println(ui.Warn(" ! an agent wrote outside its isolated copy — discarding stray changes in the base folder")) - _ = fsdiff.Restore(g.dir, g.snap) - return true -} diff --git a/docs/superpowers/specs/2026-09-26-dashboard-parallel-design.md b/docs/superpowers/specs/2026-09-26-dashboard-parallel-design.md new file mode 100644 index 0000000..978e03a --- /dev/null +++ b/docs/superpowers/specs/2026-09-26-dashboard-parallel-design.md @@ -0,0 +1,82 @@ +# Parallel runs in the dashboard — design + +Date: 2026-09-26 · Base: `feat/dashboard-ui` · Scope: `internal/tui` (Chat tab only), a new shared +package `internal/parallel` extracted from `cmd/orchestra/do.go`, README. + +## Intent + +`orchestra do --parallel` already plans a request into dependent steps, runs each dependency wave +concurrently in isolated trees and reviews + merges each result. The dashboard can only run one task at a +time. Bring the parallel workflow into Chat, with a live per-task view and the existing file-by-file +review, without a second implementation of the wave logic. + +Non-goals: changing `do --parallel` output or behaviour; touching History / Benchmarks / Agents views; +plan editing in the TUI; new dependencies. + +## Shared package: `internal/parallel` + +Pure, UI-free pieces lifted out of `do.go` (which keeps its exact prompts and output): + +- `DefaultJobs` (4) — the `--jobs` default, also the dashboard's concurrency bound. +- `StepID`, `StepTask`, `StepAgent`, `HealthyAgentNames` — step naming, prompt, per-step agent choice. +- `Graph` — scheduler nodes built from a plan (validated), with `done`/`dead` bookkeeping: + `Ready()`, `MarkDone`, `MarkDead`, `Blocked()`, `Merged()`, and `WavesLeft()` (how many more waves + if everything still pending succeeds — used for the "wave i/N" header). +- `BaseGuard` — the stray-write guard (`Arm` before a wave, `Check` after: discards anything written + into the base tree/folder, reports whether it did). The CLI prints the same warning it did before. +- `RunStep` — the quiet per-task pipeline for UIs: `engine.Produce` in the isolated tree, then commit + the result on the task branch when the tree is a git worktree (so `Isolator.Merge` can merge it). + `do --parallel` keeps using `engine.ExecuteHeadless` (it prints). + +## Dashboard flow + +Entry: a Chat message `/parallel ` (alias `/par`). The single-run path is unchanged. + +1. **Plan** (worker goroutine): in a git repo the tree must be clean (same rule as single runs); the + default agent plans with `planner.MakeParallel`; the plan is validated and `worktree.New(dir)` picks + git worktrees or folder copies. The plan is shown in the transcript and runs immediately — nothing is + kept without review anyway. +2. **Wave** (worker goroutine): the guard is armed, a tree is added per ready step (sequentially — git + worktree creation is not concurrency-safe), then `scheduler.Bounded(DefaultJobs)` runs `RunStep` per + task. Each task's `OnEvent`/`Output` is tagged with its step id and forwarded over the wave's channel + (`parEventMsg`, `parOutputMsg`, `parTaskDoneMsg`); after fan-in the guard check runs and + `parWaveDoneMsg` closes the wave. Only `Update` mutates model state. +3. **Review** (model): failed tasks are reported with their error and skipped (dead); no-change tasks + count as done; each task with changes opens the existing reviewer on `Isolator.Diff`. `y` → + `Isolator.Merge` (a conflict is reported and counts as rejected), `n` → discard. Every tree is + `Remove`d after its decision. The reviewer summary shows `wave i/N · task k/M`. +4. **Next wave** starts when all tasks are decided; when nothing is ready, blocked steps are listed, the + isolator is cleaned up and a summary line closes the run. + +Every task outcome is recorded to memory like single runs (`accepted`, `rejected`, `no-change`, +`failed`, `cancelled`), with the task's step prompt and diff, so History and history-aware routing see it. + +## Live view + +The run panel (side panel on wide terminals, stacked panel on narrow ones) becomes a task list for the +current wave. Each task is two lines: marker, step id, title, and right-aligned `status · agent · +attempt n/N · elapsed`; below it, the last line of the task's output. Status is one of queued, running, +validating, retrying, done, failed (then merged / rejected / conflict / no changes / cancelled after +review). When the row is too narrow the agent and attempt are dropped before the title is truncated. + +Keys while a wave runs: `↑/↓` or `j/k` select, `enter`/`tab` expand the selected task's full streamed +output into a full-body viewport (`↑↓ pgup pgdn` scroll), `esc` collapses; `esc` again cancels the run. +`shift+tab` still switches tabs. The status bar shows `wave i/N`. + +## Cancellation + +`esc` (or `ctrl+c`, which then quits) cancels the wave context; the worker waits for every task to stop, +runs the guard check (restoring stray writes into the base), and reports. The model then cleans up the +isolator (all temp trees), records running/queued tasks as cancelled, and — for `ctrl+c` — quits. +`ctrl+c` during a parallel review cleans up the isolator before quitting; already merged tasks stay +(they were explicitly accepted). + +## Tests (headless, fake shell agents) + +- task list rendering at several widths (fits, truncates, shows status/attempt/last line); +- event/output routing by task id updates only that row; +- select / expand / collapse keys; +- end to end in a plain temp dir and in a temp git repo: plan → wave 1 → review (accept both) → wave 2; +- conflict: two tasks editing the same file → second is reported as a conflict and not merged; +- ctrl+c mid-wave: no `orchestra-*` temp dirs remain (TMPDIR pointed at a test dir), base untouched; +- `internal/parallel`: graph waves, guard in both modes, `RunStep` commits in a worktree. diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 9d47b71..747ad4e 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -494,6 +494,10 @@ func outcomeLabel(out Outcome) string { } } +// CommitMessage is the one-line commit subject Orchestra uses for a task's +// changes ("orchestra: "). +func CommitMessage(prompt string) string { return commitMessage(prompt) } + // commitMessage builds a one-line commit subject from the task prompt. func commitMessage(prompt string) string { line := strings.TrimSpace(prompt) diff --git a/internal/parallel/parallel.go b/internal/parallel/parallel.go new file mode 100644 index 0000000..1df275d --- /dev/null +++ b/internal/parallel/parallel.go @@ -0,0 +1,191 @@ +// Package parallel holds the UI-free pieces of Orchestra's parallel workflow, +// shared by `orchestra do --parallel` and the dashboard: step naming and agent +// choice, the dependency graph that yields waves, the base-tree guard against +// agents writing outside their isolated tree, and the quiet per-task pipeline. +// +// Isolation itself lives in internal/worktree (git worktrees or folder copies) +// and the supervised loop in internal/engine; this package only wires them. +package parallel + +import ( + "context" + "fmt" + "strconv" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/fsdiff" + "github.com/crossben/orchestra-code/internal/gitutil" + "github.com/crossben/orchestra-code/internal/planner" + "github.com/crossben/orchestra-code/internal/scheduler" +) + +// DefaultJobs is the default bound on concurrently running steps. +const DefaultJobs = 4 + +// StepID is the scheduler ID of the step at index i (1-based, as planners +// number them in depends_on). +func StepID(i int) string { return strconv.Itoa(i + 1) } + +// StepTask is the prompt an agent receives for a step. +func StepTask(step planner.Step) string { + if step.Detail == "" { + return step.Title + } + return step.Title + "\n" + step.Detail +} + +// StepAgent honors a planner-assigned per-step agent when valid+healthy, +// otherwise falls back to the workflow agent. +func StepAgent(reg *agent.Registry, step planner.Step, fallback agent.Agent, fallbackName string) agent.Agent { + if step.Agent != "" && step.Agent != fallbackName { + if a, ok := reg.Get(step.Agent); ok && a.Health() == nil { + return a + } + } + return fallback +} + +// HealthyAgentNames returns the names of installed/available agents, offered +// to the planner as per-step agent choices. +func HealthyAgentNames(reg *agent.Registry) []string { + var names []string + for _, a := range reg.All() { + if a.Health() == nil { + names = append(names, a.Name()) + } + } + return names +} + +// Graph is a plan's dependency graph plus the progress made on it: steps that +// are done (merged, or finished without changes) and dead (failed, rejected, +// conflicting). Dead steps block their dependents. +type Graph struct { + nodes []scheduler.Node + done, dead map[string]bool +} + +// NewGraph builds and validates the graph for pl. +func NewGraph(pl planner.Plan) (*Graph, error) { + nodes := make([]scheduler.Node, len(pl.Steps)) + for i, s := range pl.Steps { + deps := make([]string, 0, len(s.DependsOn)) + for _, d := range s.DependsOn { + deps = append(deps, strconv.Itoa(d)) + } + nodes[i] = scheduler.Node{ID: StepID(i), Deps: deps} + } + if err := scheduler.Validate(nodes); err != nil { + return nil, err + } + return &Graph{nodes: nodes, done: map[string]bool{}, dead: map[string]bool{}}, nil +} + +// Len is the number of steps. +func (g *Graph) Len() int { return len(g.nodes) } + +// Ready returns the indices of steps that can run now. +func (g *Graph) Ready() []int { return scheduler.Ready(g.nodes, g.done, g.dead) } + +// Blocked returns the indices of steps that can never run (a dependency died). +func (g *Graph) Blocked() []int { return scheduler.Blocked(g.nodes, g.done, g.dead) } + +// MarkDone records step i as done (its dependents may run). +func (g *Graph) MarkDone(i int) { g.done[StepID(i)] = true } + +// MarkDead records step i as failed/rejected (its dependents are blocked). +func (g *Graph) MarkDead(i int) { g.dead[StepID(i)] = true } + +// Merged is the number of steps done so far. +func (g *Graph) Merged() int { return len(g.done) } + +// WavesLeft is how many waves remain, counting the one Ready would start now, +// if every pending step succeeds. It shrinks as steps die. +func (g *Graph) WavesLeft() int { + done := make(map[string]bool, len(g.done)) + for k, v := range g.done { + done[k] = v + } + n := 0 + for { + ready := scheduler.Ready(g.nodes, done, g.dead) + if len(ready) == 0 { + return n + } + n++ + for _, i := range ready { + done[StepID(i)] = true + } + } +} + +// BaseGuard catches agents that wrote into the base directory instead of their +// isolated tree (some CLIs don't honor the working directory) and discards +// that stray work. Inside a repository "stray" means the tree is no longer +// clean; in a plain folder it means the folder differs from a snapshot armed +// before the wave's fan-out. +type BaseGuard struct { + dir string + inRepo bool + snap *fsdiff.Snapshot +} + +// NewBaseGuard returns a guard for dir. +func NewBaseGuard(dir string) *BaseGuard { + return &BaseGuard{dir: dir, inRepo: gitutil.IsRepo(dir)} +} + +// InRepo reports whether the guarded directory is a git repository. +func (g *BaseGuard) InRepo() bool { return g.inRepo } + +// Arm records the base state before a wave runs (a no-op inside a +// repository, where the clean tree is the reference). +func (g *BaseGuard) Arm() error { + if g.inRepo { + return nil + } + snap, err := fsdiff.Capture(g.dir) + if err != nil { + return fmt.Errorf("snapshot %s: %w", g.dir, err) + } + g.snap = snap + return nil +} + +// Check discards any change made to the base since Arm, reporting whether it +// did. +func (g *BaseGuard) Check() bool { + if g.inRepo { + if clean, _ := gitutil.IsClean(g.dir); !clean { + _ = gitutil.Restore(g.dir) + return true + } + return false + } + if g.snap == nil { + return false + } + if d, err := fsdiff.Diff(g.dir, g.snap); err != nil || d == "" { + return false + } + _ = fsdiff.Restore(g.dir, g.snap) + return true +} + +// RunStep is the quiet per-task pipeline for UIs: dispatch → validate → +// self-correct in the isolated tree opts.Dir (engine.Produce, so OnEvent and +// Output work), then — when the tree is a git worktree and the task changed +// something — commit on the task branch so the isolator can merge it. Outside +// git the changes simply stay in the folder copy. Nothing is printed and +// nothing reaches the base tree: that is the isolator's Merge, on accept. +func RunStep(ctx context.Context, opts engine.Options) engine.Turn { + t := engine.Produce(ctx, opts) + if t.Err != nil || !t.HadChanges || ctx.Err() != nil { + return t + } + if _, err := engine.CommitAccepted(opts.Dir, engine.CommitMessage(opts.Prompt)); err != nil { + t.Err = fmt.Errorf("commit changes: %w", err) + } + return t +} diff --git a/internal/parallel/parallel_test.go b/internal/parallel/parallel_test.go new file mode 100644 index 0000000..b307a46 --- /dev/null +++ b/internal/parallel/parallel_test.go @@ -0,0 +1,179 @@ +package parallel + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/gitutil" + "github.com/crossben/orchestra-code/internal/planner" + "github.com/crossben/orchestra-code/internal/worktree" +) + +func plan(deps ...[]int) planner.Plan { + var p planner.Plan + for i, d := range deps { + p.Steps = append(p.Steps, planner.Step{Title: "s" + StepID(i), DependsOn: d}) + } + return p +} + +func TestGraphWaves(t *testing.T) { + // 1, 2 independent; 3 after 1; 4 after 3. + g, err := NewGraph(plan(nil, nil, []int{1}, []int{3})) + if err != nil { + t.Fatal(err) + } + if got := g.Ready(); len(got) != 2 || got[0] != 0 || got[1] != 1 { + t.Fatalf("wave 1 = %v", got) + } + if n := g.WavesLeft(); n != 3 { + t.Fatalf("waves left = %d, want 3", n) + } + g.MarkDone(0) + g.MarkDead(1) + if got := g.Ready(); len(got) != 1 || got[0] != 2 { + t.Fatalf("wave 2 = %v", got) + } + g.MarkDead(2) + if got := g.Ready(); len(got) != 0 { + t.Fatalf("nothing should be ready, got %v", got) + } + if b := g.Blocked(); len(b) != 1 || b[0] != 3 { + t.Fatalf("blocked = %v", b) + } + if g.Merged() != 1 || g.Len() != 4 { + t.Fatalf("merged=%d len=%d", g.Merged(), g.Len()) + } +} + +func TestStepHelpers(t *testing.T) { + if StepTask(planner.Step{Title: "t"}) != "t" || StepTask(planner.Step{Title: "t", Detail: "d"}) != "t\nd" { + t.Fatal("StepTask") + } + reg := agent.NewRegistry() + a := agent.New("alpha", "true", nil, "", nil) + reg.Add(a) + reg.Add(agent.New("beta", "true", nil, "", nil)) + reg.Add(agent.New("ghost", "no-such-binary-xyz", nil, "", nil)) + if got := StepAgent(reg, planner.Step{Agent: "beta"}, a, "alpha"); got.Name() != "beta" { + t.Fatalf("StepAgent = %s", got.Name()) + } + if got := StepAgent(reg, planner.Step{Agent: "ghost"}, a, "alpha"); got.Name() != "alpha" { + t.Fatalf("unhealthy agent should fall back, got %s", got.Name()) + } + if names := HealthyAgentNames(reg); strings.Join(names, ",") != "alpha,beta" { + t.Fatalf("healthy = %v", names) + } +} + +func gitInit(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + dir := t.TempDir() + for _, args := range [][]string{ + {"init", "-q"}, {"config", "user.email", "t@t"}, {"config", "user.name", "t"}, + {"config", "commit.gpgsign", "false"}, + } { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + os.WriteFile(filepath.Join(dir, "base.txt"), []byte("base\n"), 0o644) + for _, args := range [][]string{{"add", "-A"}, {"commit", "-qm", "init"}} { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + return dir +} + +func TestBaseGuardPlainFolder(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "base.txt"), []byte("base\n"), 0o644) + g := NewBaseGuard(dir) + if g.InRepo() { + t.Fatal("plain folder reported as repo") + } + if g.Check() { + t.Fatal("unarmed guard should not discard anything") + } + if err := g.Arm(); err != nil { + t.Fatal(err) + } + if g.Check() { + t.Fatal("clean folder should not trip the guard") + } + os.WriteFile(filepath.Join(dir, "stray.txt"), []byte("x"), 0o644) + if !g.Check() { + t.Fatal("stray write should trip the guard") + } + if _, err := os.Stat(filepath.Join(dir, "stray.txt")); !os.IsNotExist(err) { + t.Fatal("stray write should be discarded") + } +} + +func TestBaseGuardRepo(t *testing.T) { + dir := gitInit(t) + g := NewBaseGuard(dir) + if !g.InRepo() { + t.Fatal("repo not detected") + } + _ = g.Arm() + os.WriteFile(filepath.Join(dir, "stray.txt"), []byte("x"), 0o644) + if !g.Check() { + t.Fatal("stray write should trip the guard") + } + if clean, _ := gitutil.IsClean(dir); !clean { + t.Fatal("repo should be clean again") + } +} + +// RunStep in a git worktree commits the result on the task branch, so the +// isolator can merge it; in a folder copy the changes stay as files. +func TestRunStepCommitsInWorktree(t *testing.T) { + for _, mode := range []string{"git", "plain"} { + t.Run(mode, func(t *testing.T) { + var dir string + if mode == "git" { + dir = gitInit(t) + } else { + dir = t.TempDir() + } + iso, err := worktree.New(dir) + if err != nil { + t.Fatal(err) + } + defer iso.Cleanup() + tree, err := iso.Add("1", "HEAD") + if err != nil { + t.Fatal(err) + } + bin := filepath.Join(t.TempDir(), "a.sh") + os.WriteFile(bin, []byte("#!/bin/sh\necho hi > new.txt\n"), 0o755) + turn := RunStep(context.Background(), engine.Options{ + Agent: agent.New("a", bin, nil, "", nil), Prompt: "write new.txt", Dir: tree.Dir, + }) + if turn.Err != nil || !turn.HadChanges { + t.Fatalf("turn: err=%v changes=%v", turn.Err, turn.HadChanges) + } + d, err := iso.Diff(tree) + if err != nil || !strings.Contains(d, "new.txt") { + t.Fatalf("isolator diff missing new.txt: %v\n%s", err, d) + } + if conflict, err := iso.Merge(tree, "merge"); err != nil || conflict { + t.Fatalf("merge: conflict=%v err=%v", conflict, err) + } + if b, _ := os.ReadFile(filepath.Join(dir, "new.txt")); string(b) != "hi\n" { + t.Fatalf("merged file = %q", b) + } + }) + } +} From eb3416b5c3a7ddffa0b393c999f60229e119586f Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:16:57 +0000 Subject: [PATCH 22/25] tui: /parallel runs in the dashboard chat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Chat message starting with /parallel (or /par) plans the request with the default agent, then runs it wave by wave: each ready step in its own isolated tree (worktree.New: git worktrees in a repo, folder copies elsewhere), up to parallel.DefaultJobs at a time, via parallel.RunStep. - Live view: the run panel becomes a task list (status, agent, attempt, elapsed, last output line); up/down or j/k select, enter/tab expand the task's full output, esc collapses (esc on the list cancels). Worker events/output are tagged with the task id over the channel. - Review: after each wave, tasks with changes open in the existing reviewer headed "wave i/N · task k/M"; y merges (Isolator.Merge; a conflict is reported and counts as rejected), n discards. Failed tasks are reported and skipped; dependents of dead steps are skipped. - ctrl+c mid-run cancels all tasks, restores stray base writes (guard), removes every isolated tree, then quits. - Every task outcome is recorded to memory like single runs. - README documents /parallel. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 + internal/tui/chat.go | 51 ++- internal/tui/chrome.go | 10 + internal/tui/live.go | 9 +- internal/tui/par.go | 864 +++++++++++++++++++++++++++++++++++++++ internal/tui/par_test.go | 458 +++++++++++++++++++++ internal/tui/review.go | 6 +- internal/tui/tui.go | 25 +- 8 files changed, 1435 insertions(+), 8 deletions(-) create mode 100644 internal/tui/par.go create mode 100644 internal/tui/par_test.go diff --git a/README.md b/README.md index 801941a..073a373 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,25 @@ diff, validation badges, and the failing check's output if there is one. `[` / ` shows all files, `↑↓` scrolls, then `y` accepts (commits in a git repo) or `n` rejects (reverts). With the AI router enabled, each message is routed to the best agent, and plain questions are answered inline. +**Parallel runs.** Start a message with `/parallel` (or `/par`) to run it like `orchestra do --parallel`: + +```text +/parallel add a login page, a signup page, and tests for both +``` + +The default agent plans the request into steps with dependencies, then each **wave** of independent steps +runs concurrently (up to 4 at a time), every step in its own isolated tree — a git worktree inside a +repository, a folder copy in a plain folder — so nothing touches your files yet. The run panel becomes a +**task list**: one row per step with its status (queued, running, validating, retrying, done, failed), +agent, attempt, elapsed time and last output line. `↑↓` / `j k` select a task, `enter` or `tab` expands +its full output, `esc` collapses (and `esc` on the list cancels the run). When a wave finishes, every step +that changed something opens in the same review screen, headed `wave i/N · task k/M`: `y` merges it into +your folder, `n` discards it. A merge conflict with work already merged is reported and counts as a +rejection; failed steps are shown with their error and skipped, and steps that depend on a rejected or +failed one are skipped too. Then the next wave starts. `ctrl+c` mid-run cancels every task, removes every +isolated tree and undoes any stray write into your folder before quitting. Each step is recorded in +History like a single run. + The other tabs: - **Changes**: every diff you accepted or rejected. Reopen any of them with `enter`, even after a restart. @@ -312,6 +331,7 @@ internal/tui Bubble Tea dashboard: agents / history / benchmarks (read-o internal/router AI routing: Classifier (agent CLI or direct API) → Decision, tiered fallback internal/planner decompose a request into ordered steps (+ depends_on for parallel) internal/scheduler bounded-concurrency runner + DAG waves (cycle/blocked detection) +internal/parallel shared parallel workflow pieces (step graph, base-tree guard, quiet per-task run) ← do --parallel, dashboard /parallel internal/worktree task isolation: git worktrees in a repo, folder copies elsewhere; merge + conflict detection internal/engine supervised pipeline (dispatch → validate → retry → review) + headless mode ← run/shell/do internal/shell interactive chat REPL diff --git a/internal/tui/chat.go b/internal/tui/chat.go index 79e01b8..b6fa5c3 100644 --- a/internal/tui/chat.go +++ b/internal/tui/chat.go @@ -28,7 +28,12 @@ func (m Model) mainWidth() int { return m.width } -func (m Model) stackedPanelHeight() int { return min(14, m.bodyHeight()/2) } +func (m Model) stackedPanelHeight() int { + if m.par != nil { + return m.parStackedHeight() + } + return min(14, m.bodyHeight()/2) +} // layout sizes the transcript, input and any open reviewer to the window. func (m *Model) layout() { @@ -47,6 +52,10 @@ func (m *Model) layout() { if m.browsing { m.browse.setSize(m.width, bodyH) } + if m.par != nil { + m.par.out.Width = max(m.width-4, 10) // border + padding + m.par.out.Height = max(bodyH-3, 3) // title line + border + } } // setChatContent refreshes the transcript viewport, keeping it pinned to the @@ -61,14 +70,23 @@ func (m Model) updateChat(msg tea.KeyMsg) (tea.Model, tea.Cmd) { case chatReviewing: switch msg.String() { case "y", "Y": + if m.par != nil && m.par.phase == parReviewing { + return m.parAccept() + } return m.accept() case "n", "N": + if m.par != nil && m.par.phase == parReviewing { + return m.parReject() + } return m.reject() } var cmd tea.Cmd m.rv, cmd, _ = m.rv.update(msg) return m, cmd case chatRunning: + if m.par != nil { + return m.updateParRunning(msg) + } switch msg.String() { case "esc": if m.run != nil && !m.run.cancelled { @@ -115,13 +133,27 @@ func (m Model) submitChat() (tea.Model, tea.Cmd) { return m, nil } m.ta.Reset() + request, isPar := parPrefix(text) + if isPar && request == "" { + m.messages = append(m.messages, chatLine{role: "sys", + text: "usage: /parallel (or /par) — plans the request, runs independent steps in parallel, then you review each one"}) + m.setChatContent() + return m, nil + } m.ta.Blur() m.messages = append(m.messages, chatLine{role: "you", text: text}) m.cstate = chatRunning m.frame = 0 m.status = "" - cmd := m.startRun(text) - m.logEvent("info", "task: "+firstLine(text)) + var cmd tea.Cmd + if isPar { + cmd = m.startPar(request) + m.logEvent("info", "parallel: "+firstLine(request)) + } else { + m.par = nil + cmd = m.startRun(text) + m.logEvent("info", "task: "+firstLine(text)) + } m.layout() m.setChatContent() return m, cmd @@ -245,11 +277,19 @@ func (m Model) chatView() string { } bodyH := m.bodyHeight() mw := m.mainWidth() + if m.cstate == chatRunning && m.par != nil && m.par.expanded && len(m.par.tasks) > 0 { + return m.parExpandedView() + } var bottom string switch { + case m.cstate == chatRunning && !m.wide() && m.par != nil: + bottom = m.parPanel(mw, m.stackedPanelHeight()) case m.cstate == chatRunning && !m.wide(): bottom = m.runPanel(mw, m.stackedPanelHeight()) + case m.cstate == chatRunning && m.par != nil: + bottom = panelBox("", warnSty.Render(spin(m.frame))+dimSty.Render(" agents are working in parallel — ↑↓ select · enter expands · esc cancels"), + mw, inputHeight, faint) case m.cstate == chatRunning: bottom = panelBox("", warnSty.Render(spin(m.frame))+dimSty.Render(" the agent is working — follow along on the right · esc cancels"), mw, inputHeight, faint) @@ -262,7 +302,9 @@ func (m Model) chatView() string { return main } var side string - if m.run != nil { + if m.par != nil { + side = m.parPanel(m.sideWidth(), bodyH) + } else if m.run != nil { side = m.runPanel(m.sideWidth(), bodyH) } else { side = m.sessionCard(m.sideWidth(), bodyH) @@ -322,6 +364,7 @@ func (m Model) welcome() string { dimSty.Render("the checks run, and nothing is kept until you accept the diff."), "", keySty.Render("try ") + "add a /health endpoint with a test", + keySty.Render(" ") + "/parallel add login and signup" + dimSty.Render(" (steps run side by side)"), } if m.d.RoutingOn && m.d.Router != nil { lines = append(lines, keySty.Render(" ")+"why is the build slow?"+dimSty.Render(" (questions get answers)")) diff --git a/internal/tui/chrome.go b/internal/tui/chrome.go index 56206e3..90a8918 100644 --- a/internal/tui/chrome.go +++ b/internal/tui/chrome.go @@ -69,6 +69,10 @@ func (m Model) statusBar() string { switch { case m.status != "" && time.Since(m.statusAt) < 6*time.Second: right = dimSty.Render(m.status) + case m.cstate == chatRunning && m.par != nil && m.par.phase == parRunning: + right = warnSty.Render(fmt.Sprintf("wave %d/%d · %s", m.par.wave, m.par.waves, fmtElapsed(time.Since(m.par.start)))) + case m.cstate == chatRunning && m.par != nil: + right = warnSty.Render("planning · " + fmtElapsed(time.Since(m.par.start))) case m.cstate == chatRunning && m.run != nil: right = warnSty.Render(fmtElapsed(time.Since(m.run.start))) case m.active == tabChat && m.cstate == chatIdle && !m.browsing && !m.vp.AtBottom(): @@ -92,6 +96,12 @@ func (m Model) hints() []string { case tabChat: switch m.cstate { case chatRunning: + switch { + case m.par != nil && m.par.expanded: + return []string{"esc back", "↑↓ scroll", "shift+tab switch"} + case m.par != nil && m.par.phase == parRunning: + return []string{"↑↓ select", "enter expand", "esc cancel", "shift+tab switch"} + } return []string{"esc cancel", "pgup/pgdn scroll", "tab switch"} case chatReviewing: return append(m.rv.hints(), "tab switch") diff --git a/internal/tui/live.go b/internal/tui/live.go index 40ebf9d..cec84ca 100644 --- a/internal/tui/live.go +++ b/internal/tui/live.go @@ -163,6 +163,9 @@ type lineWriter struct { mu sync.Mutex ch chan tea.Msg partial string + // tag, if set, wraps each batch of lines (parallel runs tag them with the + // task id); nil sends a plain outputMsg. + tag func(lines []string) tea.Msg } func (w *lineWriter) Write(p []byte) (int, error) { @@ -193,8 +196,12 @@ func (w *lineWriter) emit(raw []string) { for _, l := range raw { lines = append(lines, cleanLine(l)) } + var msg tea.Msg = outputMsg{lines: lines} + if w.tag != nil { + msg = w.tag(lines) + } select { - case w.ch <- outputMsg{lines: lines}: + case w.ch <- msg: default: } } diff --git a/internal/tui/par.go b/internal/tui/par.go new file mode 100644 index 0000000..b09e333 --- /dev/null +++ b/internal/tui/par.go @@ -0,0 +1,864 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/gitutil" + "github.com/crossben/orchestra-code/internal/memory" + "github.com/crossben/orchestra-code/internal/parallel" + "github.com/crossben/orchestra-code/internal/planner" + "github.com/crossben/orchestra-code/internal/scheduler" + "github.com/crossben/orchestra-code/internal/ui" + "github.com/crossben/orchestra-code/internal/worktree" +) + +// Parallel runs (/parallel ): plan the request, run each dependency +// wave concurrently in isolated trees, then review every task that changed +// something with the regular reviewer and merge it on accept. The shared wave +// logic lives in internal/parallel; this file is the dashboard's view of it. +// +// Concurrency model: the plan and each wave run in a worker goroutine that +// only talks to the model through parRun.ch (tagged with the task id). The +// isolator and guard are handed to the worker for the duration of a wave and +// used by Update only between waves. + +// parPhase is where a parallel run is. +type parPhase int + +const ( + parPlanning parPhase = iota + parRunning + parReviewing + parFinished +) + +// taskStatus is one task's state, live and after review. +type taskStatus int + +const ( + tsQueued taskStatus = iota + tsRunning + tsValidating + tsRetrying + tsDone // finished; waiting for review + tsFailed + tsMerged + tsRejected + tsConflict + tsNoChanges + tsCancelled +) + +var taskStatusNames = [...]string{"queued", "running", "validating", "retrying", "done", "failed", + "merged", "rejected", "conflict", "no changes", "cancelled"} + +func (s taskStatus) String() string { return taskStatusNames[s] } + +// parTask is one step of the current wave. +type parTask struct { + idx int // step index in the plan + id string + title string + prompt string + agent string + ag agent.Agent + status taskStatus + attempt, max int + start, end time.Time + lines []string + turn engine.Turn + tree worktree.Tree + created bool + err error +} + +func (t *parTask) addLines(ls ...string) { + t.lines = append(t.lines, ls...) + if len(t.lines) > maxRunLines { + t.lines = t.lines[len(t.lines)-maxRunLines:] + } +} + +// last is the last non-blank output line. +func (t *parTask) last() string { + for i := len(t.lines) - 1; i >= 0; i-- { + if strings.TrimSpace(t.lines[i]) != "" { + return t.lines[i] + } + } + return "" +} + +func (t *parTask) elapsed() time.Duration { + switch { + case t.start.IsZero(): + return 0 + case t.end.IsZero(): + return time.Since(t.start) + } + return t.end.Sub(t.start) +} + +// parRun is a parallel run in flight (and, once finished, the last one shown +// in the side panel). Only Update mutates it. +type parRun struct { + ctx context.Context + cancel context.CancelFunc + ch chan tea.Msg // current phase's worker channel + request string + start time.Time + phase parPhase + + plan planner.Plan + graph *parallel.Graph + iso worktree.Isolator + guard *parallel.BaseGuard + planAgent agent.Agent + + wave, waves int + tasks []*parTask // the current wave + sel int + expanded bool + out viewport.Model // expanded task's output + + queue []int // tasks (indexes into tasks) awaiting review, in order + reviewing int // task under review (index into tasks) + cancelled bool +} + +// --- worker → model messages --- + +type parPlannedMsg struct { + plan planner.Plan + graph *parallel.Graph + iso worktree.Isolator + agent agent.Agent + err error +} +type parEventMsg struct { + id string + e engine.Event +} +type parOutputMsg struct { + id string + lines []string +} +type parTaskDoneMsg struct { + id string + turn engine.Turn + tree worktree.Tree + created bool + err error +} +type parWaveDoneMsg struct{ stray bool } + +// parPrefix recognises "/parallel " and "/par ". +func parPrefix(text string) (request string, ok bool) { + for _, p := range []string{"/parallel", "/par"} { + if text == p { + return "", true + } + if strings.HasPrefix(text, p+" ") || strings.HasPrefix(text, p+"\n") { + return strings.TrimSpace(text[len(p):]), true + } + } + return "", false +} + +// startPar begins planning request; the worker is launched immediately. +func (m *Model) startPar(request string) tea.Cmd { + ctx, cancel := context.WithCancel(m.d.Ctx) + p := &parRun{ctx: ctx, cancel: cancel, ch: make(chan tea.Msg, 4), request: request, + start: time.Now(), phase: parPlanning, out: viewport.New(40, 5)} + m.par = p + m.run = nil + go runParPlan(ctx, m.d, request, p.ch) + return tea.Batch(tickCmd(), waitRun(p.ch)) +} + +// runParPlan is the planning worker. It always delivers one parPlannedMsg, +// then closes ch. +func runParPlan(ctx context.Context, d Deps, request string, ch chan tea.Msg) { + defer close(ch) + msg := parPlannedMsg{} + defer func() { ch <- msg }() + + if gitutil.IsRepo(d.Dir) { + if clean, _ := gitutil.IsClean(d.Dir); !clean { + msg.err = errDirty + return + } + } + ag, ok := d.Reg.Get(d.DefaultAgent) + if !ok { + msg.err = fmt.Errorf("agent %q not available", d.DefaultAgent) + return + } + if err := ag.Health(); err != nil { + msg.err = fmt.Errorf("agent %q is not available: %w", d.DefaultAgent, err) + return + } + pl, err := planner.New(ag, d.Timeout) + if err != nil { + msg.err = err + return + } + plan, err := pl.MakeParallel(ctx, request, d.Dir, parallel.HealthyAgentNames(d.Reg)) + if err != nil { + msg.err = err + return + } + g, err := parallel.NewGraph(plan) + if err != nil { + msg.err = err + return + } + iso, err := worktree.New(d.Dir) + if err != nil { + msg.err = err + return + } + msg = parPlannedMsg{plan: plan, graph: g, iso: iso, agent: ag} +} + +// waveTask is what the wave worker needs to know about one task. +type waveTask struct { + id string + ag agent.Agent + prompt string +} + +// runParWave is the wave worker: arm the guard, create each task's tree +// (sequentially — git worktree creation is not concurrency-safe), run the +// tasks with bounded concurrency, check the guard, and report. Every task +// gets exactly one parTaskDoneMsg; parWaveDoneMsg is last. +func runParWave(ctx context.Context, d Deps, iso worktree.Isolator, guard *parallel.BaseGuard, tasks []waveTask, ch chan tea.Msg) { + defer close(ch) + send := func(msg tea.Msg) { + select { + case ch <- msg: + case <-ctx.Done(): + } + } + armErr := guard.Arm() + + trees := make([]worktree.Tree, len(tasks)) + errs := make([]error, len(tasks)) + for k, t := range tasks { + switch { + case armErr != nil: + errs[k] = armErr + case ctx.Err() != nil: + errs[k] = ctx.Err() + default: + trees[k], errs[k] = iso.Add(t.id, "HEAD") + } + } + + reported := make([]bool, len(tasks)) + scheduler.Bounded(ctx, parallel.DefaultJobs, len(tasks), func(k int) error { + t := tasks[k] + reported[k] = true + if errs[k] != nil { + ch <- parTaskDoneMsg{id: t.id, err: errs[k]} + return errs[k] + } + lw := &lineWriter{ch: ch, tag: func(lines []string) tea.Msg { return parOutputMsg{id: t.id, lines: lines} }} + turn := parallel.RunStep(ctx, engine.Options{ + Agent: t.ag, Prompt: t.prompt, Dir: trees[k].Dir, + Stages: d.Stages, MaxRetries: d.MaxRetries, + Timeout: d.Timeout, Principles: d.Principles, + OnEvent: func(e engine.Event) { send(parEventMsg{id: t.id, e: e}) }, + Output: lw, + }) + lw.flush() + ch <- parTaskDoneMsg{id: t.id, turn: turn, tree: trees[k], created: true} + return nil + }) + // Tasks the scheduler never started (cancelled while queued). + for k, t := range tasks { + if !reported[k] { + err := errs[k] + if err == nil { + err = context.Canceled + } + ch <- parTaskDoneMsg{id: t.id, tree: trees[k], created: errs[k] == nil, err: err} + } + } + // Anything an agent wrote into the base instead of its tree is discarded + // here — including after a cancel, so ctrl+c never strands stray edits. + ch <- parWaveDoneMsg{stray: guard.Check()} +} + +// onParMsg applies one worker message to the model. +func (m Model) onParMsg(msg tea.Msg) (tea.Model, tea.Cmd) { + p := m.par + if p == nil { + return m, nil + } + rearm := waitRun(p.ch) + switch msg := msg.(type) { + case parPlannedMsg: + return m.onPlanned(msg) + case parEventMsg: + if t := p.task(msg.id); t != nil { + t.apply(msg.e, m.d.MaxRetries+1) + } + case parOutputMsg: + if t := p.task(msg.id); t != nil { + t.addLines(msg.lines...) + if p.expanded && p.tasks[p.sel] == t { + m.refreshParOutput() + } + } + case parTaskDoneMsg: + if t := p.task(msg.id); t != nil { + t.finish(msg, p.cancelled) + m.logEvent("turn", fmt.Sprintf("step %s %s", t.id, t.status)) + } + case parWaveDoneMsg: + return m.onWaveDone(msg) + } + return m, rearm +} + +func (p *parRun) task(id string) *parTask { + for _, t := range p.tasks { + if t.id == id { + return t + } + } + return nil +} + +// apply updates a task from one engine event. +func (t *parTask) apply(e engine.Event, defMax int) { + switch e.Kind { + case engine.EventAttempt: + if t.start.IsZero() { + t.start = time.Now() + } + t.attempt, t.max = e.Attempt, e.Max + t.status = tsRunning + if e.Attempt > 1 { + t.status = tsRetrying + t.addLines(fmt.Sprintf("── retry %d/%d: self-correcting ──", e.Attempt-1, e.Max-1)) + } + case engine.EventStageStart: + t.status = tsValidating + } + if t.max == 0 { + t.max = defMax + } +} + +// finish records a task's final result (before review). +func (t *parTask) finish(msg parTaskDoneMsg, cancelled bool) { + t.end = time.Now() + t.turn, t.tree, t.created = msg.turn, msg.tree, msg.created + t.err = msg.err + if t.err == nil { + t.err = msg.turn.Err + } + switch { + case cancelled || errors.Is(t.err, context.Canceled): + t.status = tsCancelled + case t.err != nil: + t.status = tsFailed + case t.turn.ExitCode != 0: + t.status = tsFailed + t.err = fmt.Errorf("agent exited abnormally (code %d)", t.turn.ExitCode) + default: + t.status = tsDone + } +} + +func (m Model) onPlanned(msg parPlannedMsg) (tea.Model, tea.Cmd) { + p := m.par + p.iso, p.graph, p.plan, p.planAgent = msg.iso, msg.graph, msg.plan, msg.agent + switch { + case p.cancelled: + return m.finishPar("↺ parallel run cancelled — nothing was changed") + case msg.err != nil: + m.logEvent("error", "planning failed: "+msg.err.Error()) + return m.finishPar("✗ planning failed: " + msg.err.Error()) + } + p.guard = parallel.NewBaseGuard(m.d.Dir) + m.messages = append(m.messages, chatLine{role: "agent", agent: "plan · " + msg.agent.Name(), text: planText(msg.plan)}) + m.logEvent("info", fmt.Sprintf("planned %d steps", len(msg.plan.Steps))) + return m.startWave() +} + +// planText renders a plan as a markdown list for the transcript. +func planText(pl planner.Plan) string { + var b strings.Builder + fmt.Fprintf(&b, "%d step%s:\n\n", len(pl.Steps), plural(len(pl.Steps))) + for i, s := range pl.Steps { + fmt.Fprintf(&b, "%d. **%s**", i+1, s.Title) + if s.Agent != "" { + fmt.Fprintf(&b, " — %s", s.Agent) + } + if len(s.DependsOn) > 0 { + deps := make([]string, len(s.DependsOn)) + for j, d := range s.DependsOn { + deps[j] = fmt.Sprint(d) + } + fmt.Fprintf(&b, " _(after %s)_", strings.Join(deps, ", ")) + } + b.WriteString("\n") + } + return b.String() +} + +// startWave launches the next ready wave, or finishes when none is left. +func (m Model) startWave() (tea.Model, tea.Cmd) { + p := m.par + ready := p.graph.Ready() + if len(ready) == 0 { + return m.finishPar("") + } + p.wave++ + p.waves = p.wave - 1 + p.graph.WavesLeft() + p.tasks = p.tasks[:0] + p.sel, p.expanded, p.queue = 0, false, nil + wt := make([]waveTask, len(ready)) + for k, i := range ready { + step := p.plan.Steps[i] + ag := parallel.StepAgent(m.d.Reg, step, p.planAgent, p.planAgent.Name()) + t := &parTask{idx: i, id: parallel.StepID(i), title: step.Title, prompt: parallel.StepTask(step), + agent: ag.Name(), ag: ag, max: m.d.MaxRetries + 1} + p.tasks = append(p.tasks, t) + wt[k] = waveTask{id: t.id, ag: ag, prompt: t.prompt} + } + p.phase = parRunning + p.ch = make(chan tea.Msg, 256) + m.cstate = chatRunning + m.ta.Blur() + m.logEvent("turn", fmt.Sprintf("wave %d/%d: %d task(s)", p.wave, p.waves, len(ready))) + go runParWave(p.ctx, m.d, p.iso, p.guard, wt, p.ch) + m.layout() + m.setChatContent() + return m, tea.Batch(tickCmd(), waitRun(p.ch)) +} + +// onWaveDone triages the finished wave and opens the first review. +func (m Model) onWaveDone(msg parWaveDoneMsg) (tea.Model, tea.Cmd) { + p := m.par + p.expanded = false + if msg.stray { + m.messages = append(m.messages, chatLine{role: "sys", + text: "↺ an agent wrote outside its isolated tree — stray changes in the base were discarded"}) + m.logEvent("error", "stray writes in the base discarded") + } + if p.cancelled { + for _, t := range p.tasks { + if t.status != tsCancelled && t.status != tsFailed { + t.status = tsCancelled + } + if !t.start.IsZero() { + m.record(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: "cancelled", Attempts: t.attempt}) + } + } + return m.finishPar("↺ parallel run cancelled — isolated trees removed, any stray changes reverted") + } + for k, t := range p.tasks { + switch { + case t.status == tsFailed: + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("✗ step %s (%s) failed: %v — skipped", t.id, t.title, t.err)}) + p.graph.MarkDead(t.idx) + m.dropTree(t) + m.recordTask(t, "failed", "") + case !t.turn.HadChanges: + t.status = tsNoChanges + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("○ step %s (%s) made no changes", t.id, t.title)}) + p.graph.MarkDone(t.idx) + m.dropTree(t) + m.recordTask(t, "no-change", "") + default: + p.queue = append(p.queue, k) + } + } + if len(p.queue) > 0 { + ui.Notify("Orchestra", fmt.Sprintf("Wave %d done — %d task(s) to review", p.wave, len(p.queue))) + } + return m.nextReview() +} + +// nextReview opens the next queued task in the reviewer, or moves on. +func (m Model) nextReview() (tea.Model, tea.Cmd) { + p := m.par + for len(p.queue) > 0 { + k := p.queue[0] + p.queue = p.queue[1:] + t := p.tasks[k] + diff, err := p.iso.Diff(t.tree) + if err != nil || diff == "" { + if err == nil { + err = errors.New("no diff in the isolated tree") + } + t.status, t.err = tsFailed, err + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("✗ step %s (%s): %v — skipped", t.id, t.title, err)}) + p.graph.MarkDead(t.idx) + m.dropTree(t) + m.recordTask(t, "failed", "") + continue + } + t.turn.Diff = diff + p.reviewing = k + p.phase = parReviewing + rep := t.turn.Report + m.rv = newReviewer(diff, reviewMeta{ + Agent: t.agent, Task: t.prompt, Attempts: t.turn.Attempts, Max: m.d.MaxRetries + 1, Report: &rep, + Context: fmt.Sprintf("wave %d/%d · task %d/%d", p.wave, p.waves, k+1, len(p.tasks)), + }, m.width, m.bodyHeight()) + m.cstate = chatReviewing + m.layout() + m.setChatContent() + return m, nil + } + return m.startWave() +} + +// parAccept merges the task under review into the base. +func (m Model) parAccept() (tea.Model, tea.Cmd) { + p := m.par + t := p.tasks[p.reviewing] + conflict, err := p.iso.Merge(t.tree, fmt.Sprintf("Merge step %d: %s", t.idx+1, t.title)) + switch { + case err != nil: + t.status = tsFailed + p.graph.MarkDead(t.idx) + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("✗ step %s (%s): merge failed: %v", t.id, t.title, err)}) + m.recordTask(t, "failed", t.turn.Diff) + case conflict: + t.status = tsConflict + p.graph.MarkDead(t.idx) + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("✗ step %s (%s): merge conflict with work already merged — left unmerged; dependent steps will be skipped", t.id, t.title)}) + m.recordTask(t, "rejected", t.turn.Diff) + default: + t.status = tsMerged + p.graph.MarkDone(t.idx) + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("✓ step %s (%s) merged", t.id, t.title)}) + m.recordTask(t, "accepted", t.turn.Diff) + } + m.dropTree(t) + return m.nextReview() +} + +// parReject discards the task under review. +func (m Model) parReject() (tea.Model, tea.Cmd) { + p := m.par + t := p.tasks[p.reviewing] + t.status = tsRejected + p.graph.MarkDead(t.idx) + m.messages = append(m.messages, chatLine{role: "sys", + text: fmt.Sprintf("↺ step %s (%s) rejected — discarded", t.id, t.title)}) + m.recordTask(t, "rejected", t.turn.Diff) + m.dropTree(t) + return m.nextReview() +} + +func (m *Model) dropTree(t *parTask) { + if t.created && m.par.iso != nil { + _ = m.par.iso.Remove(t.tree) + t.created = false + } +} + +func (m *Model) recordTask(t *parTask, outcome, diff string) { + m.record(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: outcome, + Attempts: t.turn.Attempts, Passed: t.turn.Report.Passed(), Diff: diff}) +} + +// finishPar ends the run: note blocked steps, clean up every isolated tree, +// summarise. note overrides the summary (cancel / planning failure). +func (m Model) finishPar(note string) (tea.Model, tea.Cmd) { + p := m.par + if p.iso != nil { + p.iso.Cleanup() + } + if note == "" && p.graph != nil { + if blocked := p.graph.Blocked(); len(blocked) > 0 { + var ids []string + for _, i := range blocked { + ids = append(ids, fmt.Sprintf("%d (%s)", i+1, p.plan.Steps[i].Title)) + } + m.messages = append(m.messages, chatLine{role: "sys", + text: "○ skipped — a prerequisite was not merged: step " + strings.Join(ids, ", step ")}) + } + merged, total := p.graph.Merged(), p.graph.Len() + if merged == total { + note = fmt.Sprintf("✓ parallel run complete — all %d steps merged", total) + } else { + note = fmt.Sprintf("↺ parallel run done — %d/%d steps merged (others rejected, failed, or blocked)", merged, total) + } + } + m.messages = append(m.messages, chatLine{role: "sys", text: note}) + m.logEvent("turn", note) + p.phase = parFinished + p.expanded = false + p.cancel() + m.cstate = chatIdle + if m.active == tabChat { + m.ta.Focus() + } + m.layout() + m.setChatContent() + if m.quitting { + return m, tea.Quit + } + return m, nil +} + +// cancelPar asks every task (or the planner) to stop; the worker reports back. +func (m *Model) cancelPar(status string) { + if m.par.cancelled { + return + } + m.par.cancelled = true + m.par.cancel() + m.setStatus(status) +} + +// quitPar is ctrl+c while a parallel run waits for review: drop every +// isolated tree (merged work stays — it was accepted) and quit. +func (m Model) quitPar() (tea.Model, tea.Cmd) { + if m.par.iso != nil { + m.par.iso.Cleanup() + } + m.par.cancel() + return m, tea.Quit +} + +// updateParRunning handles keys while planning or while a wave runs. +func (m Model) updateParRunning(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + p := m.par + if p.expanded { + switch msg.String() { + case "esc", "enter", "tab": + p.expanded = false + return m, nil + case "up", "down", "pgup", "pgdown", "j", "k", "g", "G", "home", "end": + switch msg.String() { + case "g", "home": + p.out.GotoTop() + case "G", "end": + p.out.GotoBottom() + default: + var cmd tea.Cmd + p.out, cmd = p.out.Update(msg) + return m, cmd + } + } + return m, nil + } + switch msg.String() { + case "esc": + m.cancelPar("cancelling the parallel run…") + case "up", "k": + if p.sel > 0 { + p.sel-- + } + case "down", "j": + if p.sel < len(p.tasks)-1 { + p.sel++ + } + case "enter", "tab": + if len(p.tasks) > 0 { + p.expanded = true + m.layout() + m.refreshParOutput() + p.out.GotoBottom() + } + case "pgup", "pgdown": + var cmd tea.Cmd + m.vp, cmd = m.vp.Update(msg) + return m, cmd + } + return m, nil +} + +// refreshParOutput loads the selected task's output into the expanded +// viewport, following the tail when it was already at the bottom. +func (m *Model) refreshParOutput() { + p := m.par + if p.sel >= len(p.tasks) { + return + } + follow := p.out.AtBottom() || p.out.TotalLineCount() == 0 + t := p.tasks[p.sel] + lines := make([]string, 0, len(t.lines)) + for _, l := range t.lines { + lines = append(lines, ansiTruncate(l, max(p.out.Width, 1))) + } + if len(lines) == 0 { + lines = []string{dimSty.Render("waiting for output…")} + } + p.out.SetContent(strings.Join(lines, "\n")) + if follow { + p.out.GotoBottom() + } +} + +// --- rendering --- + +func taskGlyph(s taskStatus, frame int) string { + switch s { + case tsQueued: + return dimSty.Render("○") + case tsRunning, tsValidating, tsRetrying: + return warnSty.Render(spin(frame)) + case tsDone, tsMerged: + return okSty.Render("✓") + case tsNoChanges: + return dimSty.Render("○") + case tsRejected, tsCancelled: + return warnSty.Render("↺") + } + return badSty.Render("✗") +} + +func taskStatusStyle(s taskStatus) lipgloss.Style { + switch s { + case tsRunning, tsValidating, tsRetrying, tsRejected, tsCancelled: + return warnSty + case tsDone, tsMerged: + return okSty + case tsFailed, tsConflict: + return badSty + } + return dimSty +} + +// taskRow renders one task in two lines of width w: the summary, then the +// last line of its output. On narrow rows the agent and attempt are dropped +// before the title is squeezed. +func (m Model) taskRow(t *parTask, selected bool, w int) string { + marker := " " + if selected { + marker = selSty.Render("▸ ") + } + prefix := marker + taskGlyph(t.status, m.frame) + " " + dimSty.Render(t.id) + " " + st := taskStatusStyle(t.status).Render(t.status.String()) + var el string + if !t.start.IsZero() { + el = fmtElapsed(t.elapsed()) + } + full := []string{t.agent} + if t.attempt > 0 { + full = append(full, fmt.Sprintf("%d/%d", t.attempt, t.max)) + } + if el != "" { + full = append(full, el) + } + right := st + dimSty.Render(" · "+strings.Join(full, " · ")) + titleW := w - lipgloss.Width(prefix) - lipgloss.Width(right) - 1 + if titleW < 12 { + right = st + if el != "" { + right += dimSty.Render(" · " + el) + } + titleW = w - lipgloss.Width(prefix) - lipgloss.Width(right) - 1 + } + if titleW < 4 { + right = st + titleW = w - lipgloss.Width(prefix) - lipgloss.Width(right) - 1 + } + title := ansiTruncate(t.title, max(titleW, 1)) + if selected { + title = selSty.Render(title) + } + line1 := spread(prefix+title, right, w) + + sub := t.last() + switch { + case t.err != nil && (t.status == tsFailed || t.status == tsConflict): + sub = badSty.Render(ansiTruncate(t.err.Error(), max(w-4, 1))) + case sub == "" && t.status == tsQueued: + sub = dimSty.Render("waiting for a slot…") + case sub == "": + sub = dimSty.Render("…") + default: + sub = dimSty.Render(ansiTruncate(sub, max(w-4, 1))) + } + return ansiTruncate(line1, w) + "\n" + " " + sub +} + +// parPanel renders the task list for the current wave in a w×h box. +func (m Model) parPanel(w, h int) string { + p := m.par + inner := max(w-4, 10) + var head string + elapsed := fmtElapsed(time.Since(p.start)) + switch p.phase { + case parPlanning: + agentName := m.d.DefaultAgent + head = spread(warnSty.Render(spin(m.frame)+" planning")+dimSty.Render(" with ")+youSty.Render(agentName), + dimSty.Render(elapsed), inner) + case parRunning: + head = spread(warnSty.Render(spin(m.frame)+fmt.Sprintf(" wave %d/%d", p.wave, p.waves))+ + dimSty.Render(fmt.Sprintf(" · %d task%s", len(p.tasks), plural(len(p.tasks)))), dimSty.Render(elapsed), inner) + case parReviewing: + head = okSty.Render(fmt.Sprintf("● wave %d/%d", p.wave, p.waves)) + dimSty.Render(" · reviewing") + default: + merged, total := 0, 0 + if p.graph != nil { + merged, total = p.graph.Merged(), p.graph.Len() + } + head = dimSty.Render(fmt.Sprintf("finished · %d/%d steps merged", merged, total)) + } + body := []string{head} + if p.phase == parPlanning { + body = append(body, "", dimSty.Render("breaking the request into steps…")) + return panelBox("Parallel", strings.Join(body, "\n"), w, h, accent) + } + room := (h - 2 - 1 - 1) / 2 // border, title, head; two lines per task + start, end := listWindow(len(p.tasks), p.sel, max(room, 1)) + for i := start; i < end; i++ { + body = append(body, m.taskRow(p.tasks[i], i == p.sel && p.phase == parRunning, inner)) + } + return panelBox("Parallel", strings.Join(body, "\n"), w, h, accent) +} + +// parExpandedView is the selected task's full output, filling the body. +func (m Model) parExpandedView() string { + p := m.par + t := p.tasks[p.sel] + w := m.width + st := taskStatusStyle(t.status).Render(t.status.String()) + meta := []string{t.agent} + if t.attempt > 0 { + meta = append(meta, fmt.Sprintf("attempt %d/%d", t.attempt, t.max)) + } + if !t.start.IsZero() { + meta = append(meta, fmtElapsed(t.elapsed())) + } + head := taskGlyph(t.status, m.frame) + " " + headSty.Render("step "+t.id) + " " + ansiTruncate(t.title, max(w/2, 10)) + line := spread(head, st+dimSty.Render(" · "+strings.Join(meta, " · ")), w) + pane := lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(accent). + Padding(0, 1).Width(p.out.Width + 2).Height(p.out.Height).Render(p.out.View()) + return ansiTruncate(line, w) + "\n" + pane +} + +// parStackedHeight is the task list's height below the chat on narrow +// terminals: every task if it fits, at most two thirds of the body. +func (m Model) parStackedHeight() int { + want := 4 + 2*max(len(m.par.tasks), 1) // border, title, head + two lines per task + return max(min(want, m.bodyHeight()*2/3), 4) +} diff --git a/internal/tui/par_test.go b/internal/tui/par_test.go new file mode 100644 index 0000000..36e21a3 --- /dev/null +++ b/internal/tui/par_test.go @@ -0,0 +1,458 @@ +package tui + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" + + "github.com/crossben/orchestra-code/internal/agent" + "github.com/crossben/orchestra-code/internal/engine" + "github.com/crossben/orchestra-code/internal/gitutil" + "github.com/crossben/orchestra-code/internal/validate" +) + +// fakeParAgents registers alpha (default, also the planner) and beta. Both +// run the same script: a planning prompt prints planJSON, any other prompt +// runs work (a shell snippet that sees the prompt in "$*"). +func fakeParAgents(t *testing.T, planJSON, work string) *agent.Registry { + t.Helper() + bin := t.TempDir() + script := "#!/bin/sh\ncase \"$*\" in\n*'planning assistant'*)\ncat <<'PLAN'\n" + planJSON + "\nPLAN\n;;\n*)\n" + work + "\n;;\nesac\n" + reg := agent.NewRegistry() + for _, name := range []string{"alpha", "beta"} { + path := filepath.Join(bin, name+".sh") + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + reg.Add(agent.New(name, path, nil, "", []agent.Capability{agent.CapImplement, agent.CapPlan})) + } + return reg +} + +// Two independent steps, then a third that needs both. +const threeStepPlan = `[{"title":"write a","agent":"alpha","depends_on":[]}, +{"title":"write b","agent":"beta","depends_on":[]}, +{"title":"write c","agent":"alpha","depends_on":[1,2]}]` + +const writeFiles = `echo "working on $*" | tail -c 60 +case "$*" in +*'write a'*) echo a > a.txt ;; +*'write b'*) mkdir -p pkg && echo b > pkg/b.txt ;; +*'write c'*) cat a.txt pkg/b.txt > c.txt ;; +esac` + +func startPar(t *testing.T, m Model, request string) Model { + t.Helper() + m.ta.SetValue("/parallel " + request) + nm, _ := m.submitChat() + m = nm.(Model) + if m.par == nil { + t.Fatalf("/parallel should start a parallel run; messages=%+v", m.messages) + } + t.Cleanup(func() { + if m.par != nil && m.par.iso != nil { + m.par.iso.Cleanup() + } + }) + return m +} + +// drivePar feeds worker messages into the model until the run stops running +// (review, finished, or quit). It returns the last command. +func drivePar(t *testing.T, m Model) (Model, tea.Cmd) { + t.Helper() + deadline := time.After(30 * time.Second) + var cmd tea.Cmd + for m.par != nil && m.cstate == chatRunning { + select { + case msg, ok := <-m.par.ch: + if !ok { + t.Fatal("parallel worker closed its channel without a final message") + } + var nm tea.Model + nm, cmd = m.Update(msg) + m = nm.(Model) + case <-deadline: + t.Fatal("parallel run did not settle") + } + } + return m, cmd +} + +func gitRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "base.txt"), []byte("base\n"), 0o644) + for _, args := range [][]string{ + {"init", "-q"}, {"config", "user.email", "t@t"}, {"config", "user.name", "t"}, + {"config", "commit.gpgsign", "false"}, {"add", "-A"}, {"commit", "-qm", "init"}, + } { + if out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + return dir +} + +func plainDir(t *testing.T) string { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "base.txt"), []byte("base\n"), 0o644) + return dir +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +// Plan → wave 1 (two tasks) → review both → wave 2 (depends on both) → +// review → done, in a plain folder and in a git repository. +func TestParallelWavesReviewMerge(t *testing.T) { + for _, mode := range []string{"plain", "git"} { + t.Run(mode, func(t *testing.T) { + dir := plainDir(t) + if mode == "git" { + dir = gitRepo(t) + } + reg := fakeParAgents(t, threeStepPlan, writeFiles) + m := testModelIn(dir, reg, []validate.Stage{{Name: "test", Command: "true"}}, 140, 40) + m = startPar(t, m, "build a, b and c") + + m, _ = drivePar(t, m) + if m.cstate != chatReviewing { + t.Fatalf("wave 1 should end in review, state=%d messages=%+v", m.cstate, m.messages) + } + if len(m.par.tasks) != 2 || m.par.wave != 1 || m.par.waves != 2 { + t.Fatalf("wave 1: tasks=%d wave=%d/%d", len(m.par.tasks), m.par.wave, m.par.waves) + } + v := m.View() + for _, want := range []string{"REVIEW", "wave 1/2 · task 1/2", "a.txt", "y accept"} { + if !strings.Contains(v, want) { + t.Fatalf("first review missing %q\n%s", want, v) + } + } + m = press(m, "y") + if !strings.Contains(m.View(), "wave 1/2 · task 2/2") || !strings.Contains(m.View(), "pkg/b.txt") { + t.Fatalf("second review should be task 2/2 with pkg/b.txt:\n%s", m.View()) + } + m = press(m, "y") + if m.cstate != chatRunning || m.par.wave != 2 { + t.Fatalf("accepting the last task should start wave 2, state=%d wave=%d", m.cstate, m.par.wave) + } + m, _ = drivePar(t, m) + if m.cstate != chatReviewing || !strings.Contains(m.View(), "wave 2/2 · task 1/1") { + t.Fatalf("wave 2 should be reviewed:\n%s", m.View()) + } + m = press(m, "y") + if m.cstate != chatIdle || m.par.phase != parFinished { + t.Fatalf("run should be finished, state=%d phase=%d", m.cstate, m.par.phase) + } + if got := readFile(t, filepath.Join(dir, "c.txt")); got != "a\nb\n" { + t.Fatalf("c.txt = %q (wave 2 must see wave 1's merged work)", got) + } + if readFile(t, filepath.Join(dir, "base.txt")) != "base\n" { + t.Fatal("base.txt changed") + } + last := m.messages[len(m.messages)-1].text + if !strings.Contains(last, "all 3 steps merged") { + t.Fatalf("summary = %q", last) + } + if len(m.runs) != 3 { + t.Fatalf("each task should be recorded, got %d runs", len(m.runs)) + } + for _, r := range m.runs { + if r.Outcome != "accepted" || r.Diff == "" { + t.Fatalf("run not recorded as accepted with a diff: %+v", r) + } + } + if mode == "git" { + if clean, _ := gitutil.IsClean(dir); !clean { + t.Fatal("repository should be clean after merges") + } + out, _ := exec.Command("git", "-C", dir, "branch", "--list", "orchestra/*").Output() + if strings.TrimSpace(string(out)) != "" { + t.Fatalf("task branches left behind: %s", out) + } + } + if v := m.View(); !strings.Contains(v, "merged") { + t.Fatalf("finished panel should show merged tasks:\n%s", v) + } + }) + } +} + +// Two tasks edit the same file: the first merges, the second is reported as +// a conflict, counts as rejected, and lands nothing. A dependent step is +// skipped. +func TestParallelConflict(t *testing.T) { + dir := plainDir(t) + plan := `[{"title":"edit one","agent":"alpha"},{"title":"edit two","agent":"beta"}, +{"title":"after two","agent":"alpha","depends_on":[2]}]` + work := `case "$*" in +*'edit one'*) echo one > base.txt ;; +*'edit two'*) echo two > base.txt; echo t > two-only.txt ;; +*) echo after > after.txt ;; +esac` + m := testModelIn(dir, fakeParAgents(t, plan, work), nil, 120, 30) + m = startPar(t, m, "edit") + m, _ = drivePar(t, m) + m = press(m, "y", "y") + if m.cstate != chatIdle { + t.Fatalf("the dependent step should be skipped and the run finish, state=%d", m.cstate) + } + if got := readFile(t, filepath.Join(dir, "base.txt")); got != "one\n" { + t.Fatalf("base.txt = %q", got) + } + for _, f := range []string{"two-only.txt", "after.txt"} { + if _, err := os.Stat(filepath.Join(dir, f)); !os.IsNotExist(err) { + t.Fatalf("%s must not exist", f) + } + } + all := "" + for _, l := range m.messages { + all += l.text + "\n" + } + for _, want := range []string{"conflict", "skipped", "1/3 steps merged"} { + if !strings.Contains(all, want) { + t.Fatalf("transcript missing %q:\n%s", want, all) + } + } + outcomes := map[string]int{} + for _, r := range m.runs { + outcomes[r.Outcome]++ + } + if outcomes["accepted"] != 1 || outcomes["rejected"] != 1 { + t.Fatalf("outcomes = %v", outcomes) + } +} + +// A failed task is reported with its error and skipped; the others are still +// reviewed. +func TestParallelFailedTaskSkipped(t *testing.T) { + dir := plainDir(t) + plan := `[{"title":"good","agent":"alpha"},{"title":"bad","agent":"beta"}]` + work := `case "$*" in *'good'*) echo g > g.txt ;; *) echo boom; exit 3 ;; esac` + m := testModelIn(dir, fakeParAgents(t, plan, work), nil, 120, 30) + m = startPar(t, m, "x") + m, _ = drivePar(t, m) + if m.cstate != chatReviewing || !strings.Contains(m.View(), "g.txt") { + t.Fatalf("the good task should be under review:\n%s", m.View()) + } + found := false + for _, l := range m.messages { + if strings.Contains(l.text, "step 2") && strings.Contains(l.text, "failed") && strings.Contains(l.text, "code 3") { + found = true + } + } + if !found { + t.Fatalf("failed task not reported: %+v", m.messages) + } + m = press(m, "n") + if m.cstate != chatIdle { + t.Fatal("run should finish") + } + if _, err := os.Stat(filepath.Join(dir, "g.txt")); !os.IsNotExist(err) { + t.Fatal("rejected task must leave nothing") + } +} + +// ctrl+c mid-wave cancels every task, removes every isolated tree, undoes +// stray writes into the base folder, and only then quits. +func TestParallelCtrlCCleansUp(t *testing.T) { + dir := plainDir(t) + plan := `[{"title":"slow one","agent":"alpha"},{"title":"slow two","agent":"beta"}]` + work := `echo partial > half.txt; echo stray > '` + filepath.Join(dir, "stray.txt") + `'; sleep 20` + reg := fakeParAgents(t, plan, work) + tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) + + m := testModelIn(dir, reg, nil, 120, 30) + m = startPar(t, m, "slow") + // Run until both agents are working. + deadline := time.After(20 * time.Second) + for { + running := 0 + for _, tk := range m.par.tasks { + if tk.status == tsRunning { + running++ + } + } + if running == 2 { + if _, err := os.Stat(filepath.Join(dir, "stray.txt")); err == nil { + break + } + } + select { + case msg := <-m.par.ch: + nm, _ := m.Update(msg) + m = nm.(Model) + case <-time.After(50 * time.Millisecond): + case <-deadline: + t.Fatal("agents never started") + } + } + nm, cmd := m.Update(tea.KeyMsg{Type: tea.KeyCtrlC}) + m = nm.(Model) + if cmd != nil || !m.quitting { + t.Fatal("first ctrl+c should cancel, not quit yet") + } + start := time.Now() + m, cmd = drivePar(t, m) + if time.Since(start) > 10*time.Second { + t.Fatal("cancel did not stop the agents promptly") + } + if cmd == nil { + t.Fatal("should quit once cleaned up") + } + if _, ok := cmd().(tea.QuitMsg); !ok { + t.Fatal("expected a quit command") + } + entries, _ := os.ReadDir(tmp) + for _, e := range entries { + if strings.HasPrefix(e.Name(), "orchestra-") { + t.Fatalf("temp tree left behind: %s", e.Name()) + } + } + ents, _ := os.ReadDir(dir) + if len(ents) != 1 || ents[0].Name() != "base.txt" || readFile(t, filepath.Join(dir, "base.txt")) != "base\n" { + var names []string + for _, e := range ents { + names = append(names, e.Name()) + } + t.Fatalf("base folder should be untouched, has %v", names) + } + for _, r := range m.runs { + if r.Outcome != "cancelled" { + t.Fatalf("cancelled tasks should be recorded as cancelled: %+v", r) + } + } +} + +// parModel is a model showing a running wave of three tasks (no worker). +func parModel(w, h int) Model { + m := testModelIn(".", nil, nil, w, h) + ctx, cancel := context.WithCancel(context.Background()) + now := time.Now() + m.par = &parRun{ + ctx: ctx, cancel: cancel, ch: make(chan tea.Msg, 8), request: "r", phase: parRunning, + start: now, wave: 1, waves: 2, + tasks: []*parTask{ + {idx: 0, id: "1", title: "add the login handler with validation", agent: "alpha", status: tsRunning, attempt: 1, max: 3, start: now, lines: []string{"compiling handler"}}, + {idx: 1, id: "2", title: "write docs", agent: "beta", status: tsQueued}, + {idx: 2, id: "3", title: "wire the router", agent: "alpha", status: tsRetrying, attempt: 2, max: 3, start: now, lines: []string{"first line", "LASTLINE of output"}}, + }, + } + m.cstate = chatRunning + m.layout() + return m +} + +func TestParallelPanelFitsAtWidths(t *testing.T) { + for _, size := range [][2]int{{60, 16}, {80, 24}, {100, 30}, {140, 40}, {200, 50}} { + m := parModel(size[0], size[1]) + v := m.View() + lines := strings.Split(v, "\n") + if len(lines) != size[1] { + t.Fatalf("%dx%d: %d lines", size[0], size[1], len(lines)) + } + for i, l := range lines { + if w := lipgloss.Width(l); w > size[0] { + t.Fatalf("%dx%d line %d is %d wide:\n%s", size[0], size[1], i, w, l) + } + } + plain := ansi.Strip(v) + wants := []string{"wave 1/2", "running", "queued", "retrying", "LASTLINE"} + if size[1] < 20 { + wants = []string{"wave 1/2", "running"} // only the selected rows fit + } + for _, want := range wants { + if !strings.Contains(plain, want) { + t.Fatalf("%dx%d: panel missing %q\n%s", size[0], size[1], want, plain) + } + } + if size[0] >= 200 && !strings.Contains(plain, "2/3") { + t.Fatalf("%dx%d: wide rows should show the attempt\n%s", size[0], size[1], plain) + } + } +} + +func TestParallelEventRoutingByTaskID(t *testing.T) { + m := parModel(140, 40) + for _, msg := range []tea.Msg{ + parEventMsg{id: "2", e: engine.Event{Kind: engine.EventAttempt, Attempt: 1, Max: 3}}, + parOutputMsg{id: "2", lines: []string{"docs output"}}, + parEventMsg{id: "2", e: engine.Event{Kind: engine.EventAgentDone, Attempt: 1, Max: 3}}, + parEventMsg{id: "2", e: engine.Event{Kind: engine.EventStageStart, Stage: "test"}}, + } { + nm, _ := m.Update(msg) + m = nm.(Model) + } + t1, t2 := m.par.tasks[0], m.par.tasks[1] + if t2.status != tsValidating || t2.attempt != 1 || t2.start.IsZero() || t2.last() != "docs output" { + t.Fatalf("task 2 not updated: %+v", t2) + } + if t1.status != tsRunning || t1.last() != "compiling handler" { + t.Fatalf("task 1 should be untouched: %+v", t1) + } + nm, _ := m.Update(parEventMsg{id: "1", e: engine.Event{Kind: engine.EventAttempt, Attempt: 2, Max: 3}}) + if s := nm.(Model).par.tasks[0].status; s != tsRetrying { + t.Fatalf("attempt 2 should mark retrying, got %v", s) + } +} + +func TestParallelSelectExpandCollapse(t *testing.T) { + m := parModel(140, 40) + m = press(m, "down", "j") + if m.par.sel != 2 { + t.Fatalf("sel = %d, want 2", m.par.sel) + } + m = press(m, "k") + if m.par.sel != 1 { + t.Fatalf("k should move up, sel = %d", m.par.sel) + } + m = press(m, "k", "enter") + if !m.par.expanded { + t.Fatal("enter should expand the selected task") + } + v := ansi.Strip(m.View()) + if !strings.Contains(v, "compiling handler") || !strings.Contains(v, "esc back") { + t.Fatalf("expanded view should show the task's output:\n%s", v) + } + m = press(m, "esc") + if m.par.expanded || m.par.cancelled { + t.Fatal("esc should collapse without cancelling") + } + m = press(m, "tab") + if !m.par.expanded || m.active != tabChat { + t.Fatal("tab should expand while a wave runs") + } + m = press(m, "esc", "esc") + if !m.par.cancelled || m.par.ctx.Err() == nil { + t.Fatal("esc on the task list should cancel the run") + } +} + +func TestParallelUsage(t *testing.T) { + m := testModel() + m.ta.SetValue("/par ") + nm, _ := m.submitChat() + m = nm.(Model) + if m.par != nil || m.cstate != chatIdle || !strings.Contains(m.messages[len(m.messages)-1].text, "/parallel") { + t.Fatalf("empty /par should print usage, got %+v", m.messages) + } +} diff --git a/internal/tui/review.go b/internal/tui/review.go index b700447..1db6888 100644 --- a/internal/tui/review.go +++ b/internal/tui/review.go @@ -131,7 +131,8 @@ type reviewMeta struct { Report *validate.Report // nil = not available (history) Outcome string // history outcome; "" while pending review When time.Time - ReadOnly bool // no accept/reject (Changes, History) + ReadOnly bool // no accept/reject (Changes, History) + Context string // where this review sits, e.g. "wave 1/2 · task 1/3" (parallel runs) } // reviewer is the file-by-file diff review screen shared by Chat, Changes and @@ -277,6 +278,9 @@ func (r reviewer) summary() string { default: left = append(left, pill(strings.ToUpper(r.meta.Outcome), gray)) } + if r.meta.Context != "" { + left = append(left, headSty.Render(r.meta.Context)) + } if r.meta.Agent != "" { left = append(left, youSty.Render(r.meta.Agent)) } diff --git a/internal/tui/tui.go b/internal/tui/tui.go index b6a8438..9684532 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -7,8 +7,9 @@ // round it out. // // Layout: tui.go (model, messages, key routing), chat.go (chat + transcript), -// live.go (running a turn and the live run panel), review.go (diff review), -// views.go (list tabs), chrome.go (header + status bar), theme.go, table.go. +// live.go (running a turn and the live run panel), par.go (/parallel runs: +// waves, task list, per-task review), review.go (diff review), views.go +// (list tabs), chrome.go (header + status bar), theme.go, table.go. package tui import ( @@ -112,6 +113,7 @@ type Model struct { cstate chatState messages []chatLine run *liveRun // current or last run (nil before the first) + par *parRun // current or last /parallel run (nil when the last run was a single one) frame int rv reviewer // pending review (cstate == chatReviewing) pending engine.Turn // the turn under review @@ -288,6 +290,8 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil case routedMsg, eventMsg, outputMsg, turnMsg: return m.onRunMsg(msg) + case parPlannedMsg, parEventMsg, parOutputMsg, parTaskDoneMsg, parWaveDoneMsg: + return m.onParMsg(msg) case tea.KeyMsg: return m.onKey(msg) case tea.MouseMsg: @@ -303,6 +307,21 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m Model) onKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + if m.par != nil { + switch { + case msg.String() == "ctrl+c" && !m.quitting && (m.par.phase == parPlanning || m.par.phase == parRunning): + // Cancel every task; the worker restores stray writes, then the + // model removes every isolated tree and quits. + m.quitting = true + m.cancelPar("") + m.setStatus("cancelling, cleaning up and reverting before quitting… (ctrl+c again to force)") + return m, nil + case msg.String() == "ctrl+c" && m.par.phase == parReviewing: + return m.quitPar() + case msg.String() == "tab" && m.active == tabChat && m.par.phase == parRunning: + return m.updateParRunning(msg) // tab expands the selected task + } + } switch msg.String() { case "ctrl+c": // Mid-run, cancel first so the agent's partial edits are reverted, @@ -393,6 +412,8 @@ func (m Model) onMouse(msg tea.MouseMsg) Model { switch { case m.browsing: scroll(&m.browse.vp) + case m.active == tabChat && m.par != nil && m.par.expanded: + scroll(&m.par.out) case m.active == tabChat && m.cstate == chatReviewing: scroll(&m.rv.vp) case m.active == tabChat: From 94ea4bae50e51609164805c95f8ed036394d9918 Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:19:44 +0000 Subject: [PATCH 23/25] tui: record token usage for /parallel tasks The /parallel branch predates usage tracking, so its history records dropped tokens and cost. Stamp them via usageRun like single runs. Co-Authored-By: Claude Opus 5.5 --- internal/tui/par.go | 6 +++--- internal/tui/par_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/tui/par.go b/internal/tui/par.go index b09e333..cddc17c 100644 --- a/internal/tui/par.go +++ b/internal/tui/par.go @@ -466,7 +466,7 @@ func (m Model) onWaveDone(msg parWaveDoneMsg) (tea.Model, tea.Cmd) { t.status = tsCancelled } if !t.start.IsZero() { - m.record(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: "cancelled", Attempts: t.attempt}) + m.record(usageRun(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: "cancelled", Attempts: t.attempt}, t.turn.Usage)) } } return m.finishPar("↺ parallel run cancelled — isolated trees removed, any stray changes reverted") @@ -582,8 +582,8 @@ func (m *Model) dropTree(t *parTask) { } func (m *Model) recordTask(t *parTask, outcome, diff string) { - m.record(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: outcome, - Attempts: t.turn.Attempts, Passed: t.turn.Report.Passed(), Diff: diff}) + m.record(usageRun(memory.Run{Agent: t.agent, Prompt: t.prompt, Outcome: outcome, + Attempts: t.turn.Attempts, Passed: t.turn.Report.Passed(), Diff: diff}, t.turn.Usage)) } // finishPar ends the run: note blocked steps, clean up every isolated tree, diff --git a/internal/tui/par_test.go b/internal/tui/par_test.go index 36e21a3..3b9397f 100644 --- a/internal/tui/par_test.go +++ b/internal/tui/par_test.go @@ -456,3 +456,16 @@ func TestParallelUsage(t *testing.T) { t.Fatalf("empty /par should print usage, got %+v", m.messages) } } + +func TestParallelRecordsTaskUsage(t *testing.T) { + m := testModel() + task := &parTask{agent: "alpha", prompt: "p", turn: engine.Turn{Attempts: 1, + Usage: agent.Usage{InputTokens: 3000, OutputTokens: 300, CostUSD: 0.012, Known: true, Priced: true}}} + m.recordTask(task, "accepted", "diff") + if len(m.runs) != 1 || m.runs[0].TokensIn != 3000 || m.runs[0].TokensOut != 300 || m.runs[0].CostUSD != 0.012 { + t.Fatalf("parallel task not recorded with usage: %+v", m.runs) + } + if st := m.stats["alpha"]; st.TokensIn != 3000 { + t.Fatalf("agent totals not updated: %+v", st) + } +} From 6a5ddabad598b4260618a7b3258fdf61a0db326c Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:19:44 +0000 Subject: [PATCH 24/25] worktree: serialise Manager.Add and Cleanup do --parallel creates worktrees from several goroutines; appending to the tree list raced (caught by -race) and concurrent git worktree add contends on git's locks. Co-Authored-By: Claude Opus 5.5 --- internal/worktree/worktree.go | 9 +++++++++ internal/worktree/worktree_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go index 791ecb2..2849efe 100644 --- a/internal/worktree/worktree.go +++ b/internal/worktree/worktree.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" ) // Tree is one isolated worktree. @@ -29,6 +30,10 @@ type Manager struct { repo string // base repository directory root string // temp dir holding the worktrees trees []Tree // everything created, so Cleanup can reap leftovers + + // mu serialises Add and Cleanup: do --parallel creates trees from several + // goroutines, and concurrent `git worktree add` also contends on git's locks. + mu sync.Mutex } // NewManager prepares a worktree root for the given repo. @@ -47,6 +52,8 @@ func NewManager(repo string) (*Manager, error) { // Add creates a worktree branched from fromRef (e.g. "HEAD"). The branch is // named orchestra/. func (m *Manager) Add(id, fromRef string) (Tree, error) { + m.mu.Lock() + defer m.mu.Unlock() branch := "orchestra/" + id dir := filepath.Join(m.root, id) if _, err := m.git(m.repo, "worktree", "add", "-b", branch, dir, fromRef); err != nil { @@ -120,6 +127,8 @@ func (m *Manager) Remove(t Tree) error { // prunes git's records. Safe to call after a partial/interrupted run — any // worktree not already Removed is torn down here, so nothing leaks. func (m *Manager) Cleanup() { + m.mu.Lock() + defer m.mu.Unlock() for _, t := range m.trees { _, _ = m.git(m.repo, "worktree", "remove", "--force", t.Dir) _, _ = m.git(m.repo, "branch", "-D", t.Branch) diff --git a/internal/worktree/worktree_test.go b/internal/worktree/worktree_test.go index 38a3771..dc88275 100644 --- a/internal/worktree/worktree_test.go +++ b/internal/worktree/worktree_test.go @@ -1,6 +1,7 @@ package worktree import ( + "fmt" "os" "os/exec" "path/filepath" @@ -121,3 +122,29 @@ func indexOf(s, sub string) int { } return -1 } + +// do --parallel creates trees from several goroutines at once. +func TestAddConcurrent(t *testing.T) { + repo := initRepo(t) + m, err := NewManager(repo) + if err != nil { + t.Fatal(err) + } + defer m.Cleanup() + const n = 6 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func(i int) { + _, err := m.Add(fmt.Sprintf("c%d", i), "HEAD") + errs <- err + }(i) + } + for i := 0; i < n; i++ { + if err := <-errs; err != nil { + t.Fatal(err) + } + } + if len(m.trees) != n { + t.Fatalf("want %d tracked trees, got %d", n, len(m.trees)) + } +} From 6829bbf42366526bc53de1ff1ea3d4ffddac9ccd Mon Sep 17 00:00:00 2001 From: crossben Date: Sat, 26 Sep 2026 20:19:44 +0000 Subject: [PATCH 25/25] docs: dashboard /parallel and internal/parallel Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + plan.md | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b1dacbd..f7313a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,7 @@ Binary is at `bin/orchestra` (gitignored). - `internal/router/` — AI routing: classifies intent → picks agent. CLI classifier by default; optional direct-API classifier (`router.classifier`); history-aware agent resolution via the `History` interface. - `internal/planner/` — decomposes requests into ordered steps with `depends_on` for parallelism. - `internal/scheduler/` — bounded-concurrency DAG runner for parallel isolated tasks. +- `internal/parallel/` — shared plan → waves → isolated task logic (`Graph`, `BaseGuard`, `RunStep`) used by `do --parallel` and the dashboard's `/parallel`. - `internal/worktree/` — task isolation behind the `Isolator` interface: `worktree.New(dir)` picks git worktrees in a repository (branch per task, merge + conflict detection) or a `CopyIsolator` in plain folders (temp copy per task, all-or-nothing merge, conflict if a touched file changed in the source). - `internal/validate/` — build → lint → test pipeline, auto-detects toolchain, stops on first failure. - `internal/config/` — YAML config + built-in agent defaults + toolchain auto-detection. diff --git a/plan.md b/plan.md index d97506a..526e90d 100644 --- a/plan.md +++ b/plan.md @@ -243,8 +243,10 @@ The nice-to-haves that make the project stand out — build only once the core i the render). **Live runs** (2026-09): the engine emits optional progress events and streams agent output (`engine.Options.OnEvent`/`Output`), shown in a run panel; a file-by-file review screen is shared by Chat, Changes and History; diffs persist in memory so past changes reopen after a restart - (design: docs/superpowers/specs/2026-09-25-dashboard-ui-design.md). **Follow-up:** the same events could - tidy the interleaved output of `do --parallel`/`benchmark`. + (design: docs/superpowers/specs/2026-09-25-dashboard-ui-design.md). **Parallel in the dashboard ✅ (2026-09):** + `/parallel ` in Chat plans, runs each wave in isolated trees with a live task list, then reviews + task by task (design: docs/superpowers/specs/2026-09-26-dashboard-parallel-design.md). The CLI's + `do --parallel`/`benchmark` output could still use the same events to stop interleaving. - **Benchmark mode ✅ DONE:** `orchestra benchmark ""` runs the task through every available agent, each in an isolated git worktree (parallel), then ranks a leaderboard by validation → retries → speed → diff size, offers to merge the winner, and records results to SQLite (`benchmarks` table)