Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions internal/acp/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/Gitlawb/zero/internal/agent"
"github.com/Gitlawb/zero/internal/config"
"github.com/Gitlawb/zero/internal/modelregistry"
"github.com/Gitlawb/zero/internal/providercatalog"
"github.com/Gitlawb/zero/internal/providermodelcatalog"
"github.com/Gitlawb/zero/internal/providermodeldiscovery"
Expand Down Expand Up @@ -249,6 +250,13 @@ func (a *Agent) runTurn(ctx context.Context, sess *acpSession, userText string,
}
note := &notifier{conn: a.conn, sessionID: sess.id}

supportsVision := func(modelID string) bool {
return a.modelSupportsVision(ctx, resolved.Provider, modelID)
}
if len(images) > 0 && !supportsVision(resolved.Provider.Model) {
images = nil
}

opts := agent.Options{
Cwd: sess.cwd,
SessionID: sess.id,
Expand All @@ -259,6 +267,7 @@ func (a *Agent) runTurn(ctx context.Context, sess *acpSession, userText string,
PermissionMode: sess.currentMode(),
MaxTurns: resolved.MaxTurns,
Images: images,
SupportsVision: supportsVision,
OnText: note.text,
OnReasoning: note.thought,
OnToolCall: note.toolCall,
Expand Down Expand Up @@ -767,3 +776,32 @@ func (s *acpSession) snapshotHistory() []turnRecord {
defer s.mu.Unlock()
return append([]turnRecord(nil), s.history...)
}

func (a *Agent) modelSupportsVision(ctx context.Context, profile config.ProviderProfile, modelID string) bool {
trimmed := strings.TrimSpace(modelID)
if trimmed == "" {
return false
}
reg, _ := modelregistry.DefaultRegistry()
if entry, known := reg.Resolve(trimmed); known {
return entry.Supports(modelregistry.ModelCapabilityVision)
}
if a.deps.DiscoverModels != nil {
if discovered, err := a.deps.DiscoverModels(ctx, profile); err == nil {
for _, dm := range discovered {
if strings.EqualFold(strings.TrimSpace(dm.ID), trimmed) {
if len(dm.InputModalities) > 0 {
for _, mod := range dm.InputModalities {
if strings.EqualFold(strings.TrimSpace(mod), "image") {
return true
}
}
return false
}
break
}
}
}
}
return modelregistry.SupportsVision(reg, trimmed)
}
42 changes: 42 additions & 0 deletions internal/acp/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,48 @@ func TestACPRunTurnWiresSandboxAndScopedRegistry(t *testing.T) {
}
}

func TestACPWiresSupportsVision(t *testing.T) {
deps := testDeps(t)
var captured agent.Options
deps.RunAgent = func(_ context.Context, _ string, _ zeroruntime.Provider, opts agent.Options) (agent.Result, error) {
captured = opts
return agent.Result{FinalAnswer: "done"}, nil
}
deps.DiscoverModels = func(_ context.Context, _ config.ProviderProfile) ([]providermodeldiscovery.Model, error) {
return []providermodeldiscovery.Model{
{ID: "custom-vision", InputModalities: []string{"text", "image"}},
{ID: "custom-text", InputModalities: []string{"text"}},
}, nil
}
h := newHarness(t, deps)
defer h.stop()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

var newRes NewSessionResult
if err := h.client.Call(ctx, MethodSessionNew, NewSessionParams{Cwd: t.TempDir(), McpServers: []McpServer{}}, &newRes); err != nil {
t.Fatalf("session/new: %v", err)
}
var promptRes PromptResult
if err := h.client.Call(ctx, MethodSessionPrompt, PromptParams{
SessionID: newRes.SessionID,
Prompt: []ContentBlock{
{Type: "text", Text: "hello"},
},
}, &promptRes); err != nil {
t.Fatalf("session/prompt: %v", err)
}
if captured.SupportsVision == nil {
t.Fatal("SupportsVision was not wired into agent.Options")
}
if !captured.SupportsVision("custom-vision") {
t.Fatal("SupportsVision(custom-vision) = false, want true")
}
if captured.SupportsVision("custom-text") {
t.Fatal("SupportsVision(custom-text) = true, want false")
}
}

// TestACPRejectsInvalidCwd confirms session/new fails when the workspace root
// resolver rejects the client cwd (e.g. filesystem root).
func TestACPRejectsInvalidCwd(t *testing.T) {
Expand Down
73 changes: 52 additions & 21 deletions internal/agent/loop.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/Gitlawb/zero/internal/execution"
"github.com/Gitlawb/zero/internal/hooks"
"github.com/Gitlawb/zero/internal/modelregistry"
"github.com/Gitlawb/zero/internal/redaction"
"github.com/Gitlawb/zero/internal/sandbox"
"github.com/Gitlawb/zero/internal/streamjson"
Expand Down Expand Up @@ -646,9 +647,25 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
// recorded. They travel as user messages, and a user message between two
// tool_results breaks strict provider replay — Anthropic coalesces them
// into one user block list and requires the tool_result blocks first, so
// interleaving yields [tool_result, text, image, tool_result] and a 400.
// Same reason the self-correction feedback below is deferred.
var toolImageMessages []zeroruntime.Message
// Images ride a following USER message rather than the tool result
// above. Every provider drops images on a tool-role message —
// Anthropic's tool_result content is a string, Gemini's is a
// functionResponse, and OpenAI guards its image parts to the user role
// — so attaching them there would silently deliver nothing. A separate
// message also keeps the one-tool-result-per-tool-call pairing intact,
// which the providers validate. Vision-gate evaluation is deferred until
// after any model switch this turn resolves, so images can reach an
// escalated vision-capable model.
var toolResultsWithImages []ToolResult
buildToolImageMessages := func() []zeroruntime.Message {
var out []zeroruntime.Message
for _, tr := range toolResultsWithImages {
if imageMessage, ok := toolResultImageMessage(tr, options); ok {
out = append(out, imageMessage)
}
}
return out
}
// Parallel read-ahead state: results for calls[precomputedStart:precomputedEnd]
// executed concurrently, consumed strictly in order below.
var precomputed []precomputedToolResult
Expand Down Expand Up @@ -705,15 +722,8 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
IsError: toolResult.Status == tools.StatusError,
ChangedFiles: append([]string(nil), toolResult.ChangedFiles...),
})
// Images ride a following USER message rather than the tool result
// above. Every provider drops images on a tool-role message —
// Anthropic's tool_result content is a string, Gemini's is a
// functionResponse, and OpenAI guards its image parts to the user role
// — so attaching them there would silently deliver nothing. A separate
// message also keeps the one-tool-result-per-tool-call pairing intact,
// which the providers validate.
if imageMessage, ok := toolResultImageMessage(toolResult); ok {
toolImageMessages = append(toolImageMessages, imageMessage)
if len(toolResult.Images) > 0 {
toolResultsWithImages = append(toolResultsWithImages, toolResult)
}

// A tool may demand the run ABORT — a canceled/timed-out ask_user prompt
Expand All @@ -725,13 +735,13 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
}
if abortErr != nil {
messages = appendAbortedToolResults(messages, collected.ToolCalls[index+1:])
messages = append(messages, toolImageMessages...)
messages = append(messages, buildToolImageMessages()...)
result.Messages = copyMessages(messages)
return result, abortErr
}
if stopReason := stopReasonFromToolResult(toolResult); stopReason != "" {
messages = appendAbortedToolResults(messages, collected.ToolCalls[index+1:])
messages = append(messages, toolImageMessages...)
messages = append(messages, buildToolImageMessages()...)
result.FinalAnswer = toolResult.ModelOutput()
result.StopReason = stopReason
result.Messages = copyMessages(messages)
Expand All @@ -755,7 +765,7 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
// messages stay valid for a strict provider replay (Anthropic
// rejects a tool_use with no answering tool_result).
messages = appendAbortedToolResults(messages, collected.ToolCalls[index+1:])
messages = append(messages, toolImageMessages...)
messages = append(messages, buildToolImageMessages()...)
result.FinalAnswer = toolFailureStopAnswer(call.Name, outcome.Count)
result.Messages = copyMessages(messages)
return result, nil
Expand All @@ -774,10 +784,6 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
postEditDiagnostics.enqueue(ctx, toolResult.ChangedFiles)
}
}
// Every tool_result for this turn is now recorded, including aborted
// placeholders, so the images can follow without splitting them.
messages = append(messages, toolImageMessages...)
toolImageMessages = nil

// Run post-edit self-correction once over the union of files this turn
// changed, then append any feedback after every tool_result is recorded so
Expand Down Expand Up @@ -865,6 +871,12 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
}
}

// Every tool_result, self-correct notice, and model-switch update for this
// turn is now in place; evaluate and append image messages against the
// effective (potentially escalated) model.
messages = append(messages, buildToolImageMessages()...)
toolResultsWithImages = nil

// A turn can mix valid tool calls with a dropped (nameless) one. The valid
// calls executed above; surface the dropped call too so it is never
// silently ignored just because the turn also did real work. This is
Expand Down Expand Up @@ -3457,8 +3469,10 @@ func copyMessages(messages []Message) []Message {
//
// The text names the tool so the model can tell which call an image came from
// when several ran in one turn — the images arrive detached from their tool
// result, so nothing else associates them.
func toolResultImageMessage(result ToolResult) (zeroruntime.Message, bool) {
// result, so nothing else associates them. If the effective model cannot
// accept images, the tool's text result is left unchanged and this follow-up
// is a notice with no image parts, matching the CLI/TUI vision gate.
func toolResultImageMessage(result ToolResult, options Options) (zeroruntime.Message, bool) {
images := make([]zeroruntime.ImageBlock, 0, len(result.Images))
for _, image := range result.Images {
if len(image.Data) == 0 {
Expand All @@ -3479,9 +3493,26 @@ func toolResultImageMessage(result ToolResult) (zeroruntime.Message, bool) {
if label == "" {
label = "tool"
}
if !modelAcceptsToolImages(options) {
return zeroruntime.Message{
Role: zeroruntime.MessageRoleUser,
Content: "Image output from " + label + " was not sent because the current model does not support image input.",
}, true
}
return zeroruntime.Message{
Role: zeroruntime.MessageRoleUser,
Content: "Image output from " + label + ":",
Images: images,
}, true
}

func modelAcceptsToolImages(options Options) bool {
if options.SupportsVision != nil {
return options.SupportsVision(options.Model)
}
registry, err := modelregistry.DefaultRegistry()
if err != nil {
return modelregistry.VisionCapableByName(options.Model)
}
return modelregistry.SupportsVision(registry, options.Model)
}
Loading
Loading