Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions internal/agent/guardrails.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package agent

import (
"encoding/json"
"fmt"
"strconv"
"strings"

Expand Down Expand Up @@ -284,12 +285,67 @@ const (
noOutputStopSuffix = "to avoid consuming tokens without making progress."
)

// providerEmptyStream reports that a cleanly-completed turn carried NOTHING —
// no visible text, no tool calls (real or dropped), and no reasoning signal.
// That shape is a transport/backend fault (e.g. the ollama cloud relay
// answering 200 with an instantly-empty SSE under load), not model behavior:
// a model that is generating always leaves SOME trace (text, a tool call
// attempt, or reasoning deltas).
func providerEmptyStream(collected zeroruntime.CollectedStream) bool {
return strings.TrimSpace(collected.Text) == "" &&
len(collected.ToolCalls) == 0 &&
collected.DroppedToolCalls == 0 &&
len(collected.ReasoningBlocks) == 0 &&
!collected.ReasoningEmitted
}

// allEmptyTurnsProviderEmpty reports that every strike toward the no-output
// guard was a provider-empty stream, so the stop message should name the
// backend fault rather than generic agent no-progress.
func (state *guardState) allEmptyTurnsProviderEmpty() bool {
return state.emptyTurns > 0 && state.providerEmptyTurns == state.emptyTurns
}

// noOutputStopAnswer is the final answer returned when the no-output guard
// stops the run. The turn count is interpolated at the call site.
func noOutputStopAnswer(turns int) string {
return noOutputStopPrefix + strconv.Itoa(turns) + " turns " + noOutputStopMarker + " " + noOutputStopSuffix
}

// providerEmptyStopAnswer is the final answer when every guard strike was a
// provider-empty stream: the backend repeatedly answered with contentless
// completions (already retried with backoff inside each turn). It tells the
// user the truth — the provider failed, not the agent — and what to do next.
// detail carries wire-level diagnostics from the last empty attempt
// (finish reason, token counts) so the condition is debuggable straight from
// the session log — this exact failure class previously took a live traffic
// capture to diagnose because nothing about the response shape was recorded.
func providerEmptyStopAnswer(turns int, detail string) string {
answer := providerEmptyStopPrefix +
strconv.Itoa(turns) + " consecutive attempts, each already retried with backoff. " +
"The backend is likely rate-limiting or degraded right now — try again shortly, or switch providers/models (zero providers use <name>)."
if detail != "" {
answer += " [last attempt: " + detail + "]"
}
return answer
}

// emptyStreamDetail renders the wire-level shape of an empty attempt for
// providerEmptyStopAnswer.
func emptyStreamDetail(collected zeroruntime.CollectedStream) string {
finish := collected.FinishReason
if finish == "" {
finish = "stop"
}
return fmt.Sprintf("finish_reason=%s, output_tokens=%d, reasoning_tokens=%d",
finish, collected.Usage.OutputTokens, collected.Usage.ReasoningTokens)
}

// providerEmptyStopPrefix is the stable head of providerEmptyStopAnswer, used
// by IsNoProgressStop so session titling / resume filtering treat the
// provider-empty stop like the other guard answers (not real content).
const providerEmptyStopPrefix = "The model provider returned an empty response (no text, no tool calls, no reasoning) on "

// IsNoProgressStop reports whether content IS the no-output guardrail stop answer
// (a run that produced no visible text and no tool calls). It matches the EXACT
// structure noOutputStopAnswer emits — prefix + "<int> turns " + marker + " " +
Expand All @@ -300,6 +356,15 @@ func noOutputStopAnswer(turns int) string {
// and skip its title generation.
func IsNoProgressStop(content string) bool {
trimmed := strings.TrimSpace(content)
// The provider-empty variant: stable prefix + a bare integer count.
if rest, ok := strings.CutPrefix(trimmed, providerEmptyStopPrefix); ok {
if sep := strings.Index(rest, " consecutive attempts"); sep > 0 {
if _, err := strconv.Atoi(rest[:sep]); err == nil {
return true
}
}
return false
}
if !strings.HasPrefix(trimmed, noOutputStopPrefix) {
return false
}
Expand Down Expand Up @@ -352,6 +417,7 @@ func toolOnlyProgressReminder(turns int) string {
// purely from tool-call names and per-turn output, matching what the loop holds.
type guardState struct {
emptyTurns int
providerEmptyTurns int
totalToolCalls int
toolCallsSincePlanUpdate int
planEverCalled bool
Expand Down Expand Up @@ -417,8 +483,18 @@ func (state *guardState) observeTurn(collected zeroruntime.CollectedStream) (sto

if hasToolCalls || hasVisibleText {
state.emptyTurns = 0
state.providerEmptyTurns = 0
} else {
state.emptyTurns++
// Track separately whether the empty turn was a PROVIDER-empty stream
// (nothing at all came back — not even reasoning) vs a behavioral empty
// (the model thought/streamed but committed no answer). When every
// strike was provider-empty, the stop message names the backend fault
// instead of blaming the agent — the difference between the user
// retrying/switching providers and filing a "the agent gives up" bug.
if providerEmptyStream(collected) {
state.providerEmptyTurns++
}
}
if hasToolCalls && !hasVisibleText {
state.toolOnlyTurns++
Expand Down
127 changes: 116 additions & 11 deletions internal/agent/guardrails_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,23 @@ import (
"github.com/Gitlawb/zero/internal/zeroruntime"
)

// emptyTurn is a stream that produces no visible text and no tool calls.
// emptyTurn is a PROVIDER-empty stream: it completes cleanly carrying nothing
// at all (the ollama-cloud 200+[DONE] shape). The loop retries these in-turn
// before counting a no-output strike.
func emptyTurn() []zeroruntime.StreamEvent {
return []zeroruntime.StreamEvent{{Type: zeroruntime.StreamEventDone}}
}

// reasoningOnlyTurn is a BEHAVIORAL empty turn: the model streamed reasoning
// (so the provider clearly worked) but committed no text and no tool calls.
// These are NOT retried in-turn; they count directly toward the no-output guard.
func reasoningOnlyTurn() []zeroruntime.StreamEvent {
return []zeroruntime.StreamEvent{
{Type: zeroruntime.StreamEventReasoning, Content: "thinking…"},
{Type: zeroruntime.StreamEventDone},
}
}

// textTurn produces a turn with visible assistant text.
func textTurn(content string) []zeroruntime.StreamEvent {
return []zeroruntime.StreamEvent{
Expand Down Expand Up @@ -45,9 +57,9 @@ func countUserMessagesContaining(messages []zeroruntime.Message, needle string)
func TestRunStopsAfterConsecutiveEmptyTurns(t *testing.T) {
provider := &mockProvider{
turns: [][]zeroruntime.StreamEvent{
emptyTurn(),
emptyTurn(),
emptyTurn(),
reasoningOnlyTurn(),
reasoningOnlyTurn(),
reasoningOnlyTurn(),
// A 4th turn exists but must never be requested.
textTurn("should never reach here"),
},
Expand Down Expand Up @@ -77,10 +89,10 @@ func TestRunStopsAfterConsecutiveEmptyTurns(t *testing.T) {
func TestRunResetsEmptyTurnCounterOnVisibleOutput(t *testing.T) {
provider := &mockProvider{
turns: [][]zeroruntime.StreamEvent{
emptyTurn(),
emptyTurn(),
reasoningOnlyTurn(),
reasoningOnlyTurn(),
textTurn("here is real progress"), // resets the counter and is the final answer
emptyTurn(),
reasoningOnlyTurn(),
},
}

Expand Down Expand Up @@ -109,11 +121,11 @@ func TestRunResetsEmptyTurnCounterOnToolCall(t *testing.T) {

provider := &mockProvider{
turns: [][]zeroruntime.StreamEvent{
emptyTurn(),
emptyTurn(),
reasoningOnlyTurn(),
reasoningOnlyTurn(),
toolTurn("call-1", "read_file", `{"path":"notes.txt"}`), // resets counter
emptyTurn(),
emptyTurn(),
reasoningOnlyTurn(),
reasoningOnlyTurn(),
textTurn("done"),
},
}
Expand Down Expand Up @@ -455,3 +467,96 @@ func TestRunInjectsToolFailureHintWithSchema(t *testing.T) {
t.Fatalf("expected a tool-failure hint on the 3rd turn, messages: %+v", provider.requests[2].Messages)
}
}

// A provider-empty stream (clean completion carrying nothing at all) is
// retried in-turn with backoff before it may count as a no-output strike; when
// the backend keeps answering empty, the run stops with a message that names
// the PROVIDER fault — not the generic "agent made no progress" text. This is
// the exact live failure observed on the ollama cloud relay (HTTP 200 in ~1s,
// SSE straight to [DONE]), which silently killed 56% of sessions on the
// reporting machine.
func TestRunRetriesProviderEmptyStreamsThenStopsWithProviderMessage(t *testing.T) {
turns := make([][]zeroruntime.StreamEvent, 0, 9)
for i := 0; i < 9; i++ { // 3 strikes × (1 initial + 2 in-turn retries)
turns = append(turns, emptyTurn())
}
provider := &mockProvider{turns: turns}

result, err := Run(context.Background(), "go", provider, Options{
Registry: tools.NewRegistry(),
MaxTurns: 12,
})
if err != nil {
t.Fatal(err)
}
wantRequests := maxEmptyTurns * (1 + maxEmptyStreamRetries)
if len(provider.requests) != wantRequests {
t.Fatalf("expected %d requests (%d strikes × %d attempts), got %d",
wantRequests, maxEmptyTurns, 1+maxEmptyStreamRetries, len(provider.requests))
}
if !strings.Contains(result.FinalAnswer, "provider returned an empty response") {
t.Fatalf("expected the provider-empty stop message, got %q", result.FinalAnswer)
}
if strings.Contains(result.FinalAnswer, noOutputStopMarker) {
t.Fatalf("provider fault must not be reported as generic agent no-progress: %q", result.FinalAnswer)
}
if !IsNoProgressStop(result.FinalAnswer) {
t.Fatal("the provider-empty stop answer must be recognized by IsNoProgressStop (titling/resume filters)")
}
// Wire-level diagnostics must land in the message (and therefore the
// session log) so this failure class is debuggable without a traffic capture.
if !strings.Contains(result.FinalAnswer, "output_tokens=0") || !strings.Contains(result.FinalAnswer, "finish_reason=") {
t.Fatalf("stop answer must carry last-attempt diagnostics, got %q", result.FinalAnswer)
}
}

// A transient empty response recovers on the in-turn retry: no strike, no stop,
// the retried turn's real output is the answer.
func TestRunRecoversWhenEmptyStreamRetrySucceeds(t *testing.T) {
provider := &mockProvider{
turns: [][]zeroruntime.StreamEvent{
emptyTurn(), // initial attempt: backend hiccup
textTurn("recovered"), // in-turn retry gets the real answer
},
}

result, err := Run(context.Background(), "go", provider, Options{
Registry: tools.NewRegistry(),
MaxTurns: 12,
})
if err != nil {
t.Fatal(err)
}
if len(provider.requests) != 2 {
t.Fatalf("expected 2 requests (initial + one retry), got %d", len(provider.requests))
}
if result.FinalAnswer != "recovered" {
t.Fatalf("expected the retried turn's text as final answer, got %q", result.FinalAnswer)
}
if result.Turns != 1 {
t.Fatalf("an in-turn retry is the SAME turn, want 1 turn, got %d", result.Turns)
}
}

// Mixed strikes (provider-empty + behavioral reasoning-only) must NOT claim a
// provider fault — the generic no-output message stays.
func TestRunMixedEmptyTurnsKeepGenericStopMessage(t *testing.T) {
provider := &mockProvider{
turns: [][]zeroruntime.StreamEvent{
reasoningOnlyTurn(), // strike 1: behavioral
emptyTurn(), emptyTurn(), emptyTurn(), // strike 2: provider-empty ×(1+2 retries)
reasoningOnlyTurn(), // strike 3: behavioral
},
}

result, err := Run(context.Background(), "go", provider, Options{
Registry: tools.NewRegistry(),
MaxTurns: 12,
})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(result.FinalAnswer, noOutputStopMarker) {
t.Fatalf("mixed strikes must use the generic no-output message, got %q", result.FinalAnswer)
}
}
53 changes: 52 additions & 1 deletion internal/agent/loop.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,20 @@ const maxTurnsFinalAnswerPrompt = "You have reached the tool-turn limit. Do not
// WITH NO OUTPUT yet is re-issued on a fresh connection before giving up. Only
// the no-output case is retried (a partial turn would duplicate), so this is a
// safe recovery for a stalled/dead pooled connection.
const maxStreamStallRetries = 2
// maxStreamStallRetries: 3 in-turn re-issues (4 attempts total) approaches
// Codex's stream_max_retries=5 posture while staying hard-capped (opencode's
// unbounded session retry has produced infinite loops on this same failure).
// Affordable now that a mid-stream death is detected at the gap timeout
// (~2m) instead of the full 5m idle window.
const maxStreamStallRetries = 3

// maxEmptyStreamRetries caps in-turn re-issues of a request whose stream
// completed cleanly but carried nothing at all (providerEmptyStream). Observed
// live on the ollama cloud relay under load: HTTP 200 in ~1s with an SSE that
// goes straight to [DONE]. The condition is transient backend state, so a
// short backoff-retry usually recovers; without it three such responses
// arrived inside two seconds and killed the run via the no-output guard.
const maxEmptyStreamRetries = 2

const (
toolResultMetaControl = "control"
Expand Down Expand Up @@ -324,6 +337,35 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
}
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, forwardingOpts)
}
// Provider-empty retry: the stream completed CLEANLY but carried nothing
// — no text, no tool calls, no reasoning. That is not model behavior (a
// generating model always leaves some trace); it is a backend answering
// with a contentless completion (seen on the ollama cloud relay under
// load: 200 + instant [DONE]). Transient, so re-issue with backoff before
// letting it count as a no-output strike. Nothing from the empty stream
// was forwarded or committed to history, so the retry is clean.
for attempt := 1; attempt <= maxEmptyStreamRetries &&
collected.Error == "" && !forwardedVisibleText &&
providerEmptyStream(collected); attempt++ {
if notify := emptyRetryNoticeFor(options); notify != nil {
notify(attempt, maxEmptyStreamRetries)
}
if err := sleepWithContext(ctx, backoffFor(attempt)); err != nil {
result.Messages = copyMessages(messages)
return result, err
}
retryRequest := zeroruntime.CompletionRequest{
Messages: copyMessages(messages),
Tools: exposed,
ReasoningEffort: options.ReasoningEffort,
}
retryStream, retryErr := streamWithReconnect(ctx, provider, retryRequest, reconnectNoticeFor(options))
if retryErr != nil {
result.Messages = copyMessages(messages)
return result, retryErr
}
collected = zeroruntime.CollectStreamWithOptions(ctx, retryStream, forwardingOpts)
}
if collected.Error != "" {
// Route a reissued stream's non-stall error through the SAME recovery as
// the initial stream (image-rejection wrapping / context-limit compaction)
Expand Down Expand Up @@ -370,6 +412,15 @@ func Run(ctx context.Context, prompt string, provider Provider, options Options)
// A truly-empty turn (no text, no tool calls, no dropped calls) is
// counted toward the runaway cap so we stop before burning maxTurns.
if guards.observeTurn(collected) {
// Every strike being a provider-empty stream means the BACKEND
// failed repeatedly (each attempt already retried with backoff
// above) — say so, instead of the generic no-progress message
// that reads as the agent giving up.
if guards.allEmptyTurnsProviderEmpty() {
result.FinalAnswer = providerEmptyStopAnswer(result.Turns, emptyStreamDetail(collected))
result.Messages = copyMessages(messages)
return result, nil
}
result.FinalAnswer = noOutputStopAnswer(result.Turns)
result.Messages = copyMessages(messages)
return result, nil
Expand Down
16 changes: 16 additions & 0 deletions internal/agent/reconnect.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,22 @@ func stallRetryNoticeFor(options Options) reconnectNotifier {
}
}

// emptyRetryNoticeFor builds a notifier for the loop's provider-empty retry
// (a stream that completed cleanly but carried NOTHING — no text, no tool
// calls, no reasoning). Distinct wording from the stall notice because nothing
// timed out: the backend answered instantly with an empty completion (observed
// on the ollama cloud relay under load, and possible on any gateway having a
// bad moment). Surfaced through OnReasoning, the non-content channel. Nil when
// there is no reasoning sink (the retry still happens silently).
func emptyRetryNoticeFor(options Options) reconnectNotifier {
if options.OnReasoning == nil {
return nil
}
return func(attempt, max int) {
options.OnReasoning(fmt.Sprintf("\n[provider returned an empty response — retrying %d/%d…]\n", attempt, max))
}
}

// streamWithReconnect issues request via provider.StreamCompletion and, on a
// transient disconnect error, retries the connect up to maxStreamReconnects
// times with exponential backoff. It returns the live stream on success, or the
Expand Down
Loading
Loading