Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions src/main/agent-hooks/server-ingest-structured-state-clock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// The host's status row takes its state clock from the summary's `statusStartedAt`, the session's
// own lifecycle clock, so `worktree ps`, mobile and the dashboard date a parent the way the sidebar
// does. An older summary without the clock keeps the ingest's own continuity rule.

import { beforeEach, describe, expect, it } from 'vitest'
import type { AgentSessionStatusSummary } from '../../shared/agent-session-wire'
import { makeStructuredAgentStatusSubject } from '../../shared/agent-status-subject'
import { AgentHookServer, _internals } from './server'

const SESSION = 'b2c3d4e5-f6a7-4b8c-9d0e-1f2a3b4c5d6e'
const SUBJECT = makeStructuredAgentStatusSubject(
{
executionHostId: 'local',
wslDistro: null,
workspaceId: 'repo-1::/workspace/app',
workspaceKind: 'git-worktree'
},
SESSION
)
const SETTLED = 22_000

function summary(over: Partial<AgentSessionStatusSummary> = {}): AgentSessionStatusSummary {
return {
sessionId: SESSION,
workspaceId: 'repo-1::/workspace/app',
agent: 'codex',
status: 'idle',
hostExecutionOwned: true,
latestPrompt: 'fan out',
updatedAt: SETTLED,
...over
}
}

function row(server: AgentHookServer) {
return server.getStatusSnapshot()[0]
}

beforeEach(() => {
_internals.resetCachesForTests()
})

describe("the host row's state clock", () => {
it("dates a subagent's approval at the ask, and the parent's completion where it was", () => {
const server = new AgentHookServer()
server.ingestStructuredStatus(summary({ statusStartedAt: SETTLED }), SUBJECT)
server.ingestStructuredStatus(
summary({ status: 'attention', statusStartedAt: 27_000, updatedAt: 27_000 }),
SUBJECT
)
expect(row(server)).toMatchObject({
state: 'blocked',
stateStartedAt: 27_000,
mainAgent: { state: 'blocked', stateStartedAt: 27_000 }
})

server.ingestStructuredStatus(summary({ statusStartedAt: SETTLED, updatedAt: 28_500 }), SUBJECT)
expect(row(server)).toMatchObject({
state: 'done',
stateStartedAt: SETTLED,
mainAgent: { state: 'done', stateStartedAt: SETTLED }
})
})

it('settles a row child work held open on the parent clock, not the last child row', () => {
const server = new AgentHookServer()
server.ingestStructuredStatus(
summary({ status: 'working', statusStartedAt: 10_000, updatedAt: 10_000 }),
SUBJECT
)
server.ingestStructuredStatus(
summary({
statusStartedAt: SETTLED,
updatedAt: 24_000,
backgroundTasks: [{ id: 'child-1', kind: 'agent', state: 'working' }]
}),
SUBJECT
)
expect(row(server)).toMatchObject({ state: 'working', stateStartedAt: 10_000 })

server.ingestStructuredStatus(summary({ statusStartedAt: SETTLED, updatedAt: 26_000 }), SUBJECT)
expect(row(server)).toMatchObject({ state: 'done', stateStartedAt: SETTLED })
})

it("keeps the ingest's own continuity for an older host's summary, which carries no clock", () => {
const server = new AgentHookServer()
server.ingestStructuredStatus(summary(), SUBJECT)
server.ingestStructuredStatus(summary({ status: 'attention', updatedAt: 27_000 }), SUBJECT)
server.ingestStructuredStatus(summary({ updatedAt: 28_500 }), SUBJECT)
expect(row(server)).toMatchObject({
state: 'done',
stateStartedAt: 28_500,
mainAgent: { state: 'done', stateStartedAt: 28_500 }
})
})
})
11 changes: 8 additions & 3 deletions src/main/agent-hooks/server/server-ingest-structured.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ import {
isAgentStatusHeldOpenByChildWork
} from '../../../shared/agent-lead-status-fold'
import { structuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status'
import {
structuredAgentSessionDatedMainAgent,
structuredAgentSessionRowStateStartedAt
} from '../../../shared/structured-agent-session-status-started-at'
import { structuredStatusLegacyEvent } from './server-structured-status-row'
import { AgentHookServerIngestTerminal } from './server-ingest-terminal'

Expand Down Expand Up @@ -49,7 +53,7 @@ export abstract class AgentHookServerIngestStructured extends AgentHookServerIng
// by the journal: a restart's republish is not a new main agent state either.
const mainAgent = continueMainAgentStatus(
priorStatus?.mainAgent,
agentStatus.mainAgent,
structuredAgentSessionDatedMainAgent(agentStatus.mainAgent, summary),
summary.updatedAt
)
const tabId = structuredAgentSessionTabId(parsed.sessionId)
Expand Down Expand Up @@ -88,9 +92,10 @@ export abstract class AgentHookServerIngestStructured extends AgentHookServerIng
// Continuity is the whole published work identity: `state` alone no longer means "a turn is
// running", so monitoring that becomes a real turn must restart the clock, not inherit it.
stateStartedAt:
priorStatus?.state === state && priorStatus.workingMode === workingMode
structuredAgentSessionRowStateStartedAt({ state, mainAgent }, summary) ??
(priorStatus?.state === state && priorStatus.workingMode === workingMode
? priorStatus.stateStartedAt
: summary.updatedAt,
: summary.updatedAt),
observation: {
origin: 'structured',
kind: 'transition',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export function journalRenderItem(
body,
sequence: row.seq,
observedAt: row.ts,
...(row.recovered ? { recoveredAt: row.ts } : {}),
...(row.recovered ? { recovered: row.recovered } : {}),
...agentJournalLinkageFields(row)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
// A turn that was running when its host went away ends when recovery settles it. That settlement is
// the edge the user needs to see — their work stopped — so the session reads as newly done then,
// and nothing along the way may call it a success. Every hop is the real one: durable journal,
// recovery settlement, status feed, the host's status row, and the turn-completion feed.

import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import type {
AgentSessionStatusSummary,
AgentSessionTurnCompletionEvent
} from '../../../shared/agent-session-wire'
import { AgentHookServer, _internals } from '../../agent-hooks/server'
import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open'
import type { AgentSessionJournal } from '../agent-session-journal/journal-store'
import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement'
import {
settleStaleSessionStateOnAcquire,
type StructuredAgentSessionTurnVerdict
} from './structured-agent-session-stale-turn-verdict'
import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed'
import { indexedStatusFeedSession } from './structured-agent-session-status-feed-test-session'
import { StructuredAgentSessionTurnCompletionFeed } from './structured-agent-session-turn-completion-feed'

const SESSION = 'recovered-turn-session'
const THREAD = 'thread-1'
const TURN_STARTED = 1_000
const EXIT_OBSERVED = 2_000
const RECOVERED = 9_000

let root: string
const journals = createTrackedJournalOpener()

beforeEach(async () => {
_internals.resetCachesForTests()
root = await mkdtemp(join(tmpdir(), 'orca-recovered-turn-'))
})

afterEach(async () => {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
})

/** A session whose turn was running when its host went away, reopened by the next host. */
async function sessionWithRunningTurn() {
let clock = TURN_STARTED
const journal = await journals.open({
identity: {
sessionId: SESSION,
workspaceId: 'workspace-1',
hostId: 'local',
agent: 'codex',
providerHandle: { kind: 'codex', threadId: THREAD }
},
now: () => clock,
journalDir: join(root, SESSION)
})
await journal.appendItem(
{ provider: 'orca', clientMessageId: 'prompt-1' },
{ kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'long job' }] },
{ fence: 1 }
)
await journal.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 9 },
{ kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: TURN_STARTED },
{ fence: 1 }
)
const server = new AgentHookServer()
const sessions = new Map([[SESSION, indexedStatusFeedSession({ journal })]])
const feed = new StructuredAgentSessionStatusFeed({
sessions,
getRecord: () => null,
now: () => clock,
statusSink: () => ({
publish: (summary, subject) => server.ingestStructuredStatus(summary, subject),
forget: (subject) => server.dropStructuredStatus(subject)
})
})
const summaries: AgentSessionStatusSummary[] = []
feed.subscribe({
id: 'list-1',
emit: (event) => {
if (event.type === 'snapshot') {
summaries.push(...event.sessions)
} else if (event.type === 'status') {
summaries.push(event.session)
}
}
})
const completions = new StructuredAgentSessionTurnCompletionFeed({ sessions, now: () => clock })
const completionEvents: AgentSessionTurnCompletionEvent[] = []
completions.subscribe({ id: 'dot-1', emit: (event) => completionEvents.push(event) })
// Both feeds have seen the turn running, so its settlement is a transition they must judge.
completions.observe(SESSION, journal)
expect(summaries.at(-1)).toMatchObject({ status: 'working', statusStartedAt: TURN_STARTED })
const publish = (): void => {
feed.publish(SESSION, journal)
completions.observe(SESSION, journal)
}
return {
journal,
server,
summaries,
completionEvents,
publish,
recoverAt: (at: number) => {
clock = at
}
}
}

function settleDeadGeneration(
journal: AgentSessionJournal,
verdict: StructuredAgentSessionTurnVerdict
): Promise<boolean> {
return settleStructuredAgentSessionDeadGeneration({
journal,
sessionId: SESSION,
fence: 2,
settlementId: 'settle-1',
verdict,
pendingSubmissionReason: 'provider_exited_before_acknowledgement'
})
}

describe('a turn recovery settled after its host went away', () => {
it.each([
['an unverifiable end', { state: 'unverifiable' } as const],
['an exit observed before the restart', { state: 'interrupted', completedAt: EXIT_OBSERVED }]
] satisfies [string, StructuredAgentSessionTurnVerdict][])(
'is done as of the recovery, never as a success: %s',
async (_label, verdict) => {
const session = await sessionWithRunningTurn()
session.recoverAt(RECOVERED)
expect(await settleDeadGeneration(session.journal, verdict)).toBe(true)
session.publish()

expect(session.summaries.at(-1)).toMatchObject({
status: 'idle',
statusStartedAt: RECOVERED
})
expect(session.summaries.at(-1)).not.toHaveProperty('turnOutcome')
const [row] = session.server.getStatusSnapshot()
// A done row dated at the recovery is a completion the user has not read yet.
expect(row).toMatchObject({
state: 'done',
stateStartedAt: RECOVERED,
mainAgent: { state: 'done', stateStartedAt: RECOVERED }
})
expect(row?.mainAgent).not.toHaveProperty('outcome')
// The dot and the OS notification come only from a completion event, and none is sent.
expect(session.completionEvents).toEqual([])
}
)

it('is dated the same way when a new provider child finds the turn still running', async () => {
const session = await sessionWithRunningTurn()
session.recoverAt(RECOVERED)
await settleStaleSessionStateOnAcquire({
journal: session.journal,
sessionId: SESSION,
fence: 2,
acquisitionGeneration: 'generation-2'
})
session.publish()

expect(session.summaries.at(-1)).toMatchObject({
status: 'idle',
statusStartedAt: RECOVERED
})
expect(session.server.getStatusSnapshot()[0]).toMatchObject({
state: 'done',
stateStartedAt: RECOVERED
})
expect(session.completionEvents).toEqual([])
})

// The control that keeps the silence above from being vacuous: a turn its provider finished does
// reach the completion feed through this same harness, dated by its own end.
it('leaves a turn its provider finished to the provider, dated by its own end', async () => {
const session = await sessionWithRunningTurn()
session.recoverAt(RECOVERED)
await session.journal.appendItem(
{ provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 9 },
{
kind: 'turn',
turnId: 'turn-1',
state: 'completed',
outcome: 'success',
startedAt: TURN_STARTED,
completedAt: EXIT_OBSERVED
},
{ fence: 1 }
)
session.publish()

expect(session.summaries.at(-1)).toMatchObject({
status: 'idle',
statusStartedAt: EXIT_OBSERVED,
turnOutcome: 'success'
})
expect(session.completionEvents).toEqual([
expect.objectContaining({
type: 'completion',
completion: expect.objectContaining({ outcome: 'success' })
})
])
})
})
Loading
Loading