From e5c463cc56ba5e726c5dd0c812ece598aee577b3 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Wed, 3 Jun 2026 20:28:46 +0300 Subject: [PATCH 1/6] Improve Captain browser --- docs/agents.md | 25 +- src/ai/captain.ts | 80 +++++-- src/ai/captain/idle-mode.ts | 78 +++++++ src/ai/captain/mixin.ts | 2 +- src/ai/captain/web-mode.ts | 45 +++- src/ai/historian/screencast.ts | 13 +- src/ai/pilot.ts | 28 ++- src/ai/tester.ts | 211 +++++++++++++++-- src/explorer.ts | 108 +++++++-- tests/unit/captain-mode.test.ts | 117 ++++++++++ tests/unit/explorer-recovery-url.test.ts | 60 +++++ tests/unit/historian-screencast.test.ts | 41 ++++ tests/unit/pilot-evidence.test.ts | 35 +++ tests/unit/tester-execution-recovery.test.ts | 228 +++++++++++++++++++ 14 files changed, 1000 insertions(+), 71 deletions(-) create mode 100644 tests/unit/captain-mode.test.ts create mode 100644 tests/unit/explorer-recovery-url.test.ts create mode 100644 tests/unit/historian-screencast.test.ts create mode 100644 tests/unit/pilot-evidence.test.ts create mode 100644 tests/unit/tester-execution-recovery.test.ts diff --git a/docs/agents.md b/docs/agents.md index 7ba02d2..ef88d64 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -196,15 +196,28 @@ export default { The agent uses the default model unless overridden. The report file is always written to `output/reports/`; there is no opt-out for the file itself, but `enabled: false` disables the agent so nothing runs. -## Captain Agent *(coming soon)* +## Captain Agent -**Purpose:** Orchestrates the whole testing session. +**Purpose:** Supervises explicit user requests and non-standard recovery situations. + +**Modes:** +- `idle` - plan management, project inspection, knowledge and experience file work. Available even before a page is loaded. +- `web` - page interaction, navigation, browser diagnostics, visual/context checks. +- `test` - test timeline inspection, state inspection, generated code/log analysis. +- `heal` - browser and test recovery when an active test loses its page or browser context. **What it does:** -- Coordinates all agents intelligently -- Responds to user commands in real-time -- Adjusts strategy based on discoveries -- Manages conversation context efficiently +- Handles direct TUI requests that need more judgment than a slash command +- Explains current Explorbot configuration and suggests focused setup improvements +- Reads recent output artifacts before answering questions about previous sessions +- Inspects active tests, failed steps, page states, and Pilot analysis +- Recovers closed/crashed pages during test execution and tells Tester how to continue +- Can reload, recover, restart the browser, open a fresh tab, or close extra tabs when needed + +**When Captain runs:** +- On explicit user requests in the TUI +- During test interrupts where the user asks to stop, pass, skip, or redirect execution +- During fatal browser execution errors, where it first tries recovery before stopping the test ## Per-Agent Model Configuration diff --git a/src/ai/captain.ts b/src/ai/captain.ts index ddb5548..1eb471e 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -80,14 +80,19 @@ export class Captain extends CaptainBase implements Agent { } } - private detectMode(): CaptainMode { - if (this.explorBot.getExplorer().activeTest) return 'test'; - if (this.explorBot.getExplorer().getStateManager().getCurrentState()) return 'web'; + getMode(): CaptainMode { + const explorer = this.explorBot.getExplorer(); + const activeTest = explorer.activeTest; + const page = explorer.playwrightHelper?.page; + + if (activeTest && (!page || page.isClosed?.())) return 'heal'; + if (activeTest) return 'test'; + if (explorer.getStateManager().getCurrentState()) return 'web'; return 'idle'; } private systemPrompt(): string { - const mode = this.detectMode(); + const mode = this.getMode(); const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url; const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl); @@ -101,18 +106,20 @@ export class Captain extends CaptainBase implements Agent { - idle: plan management, file operations, knowledge. Always available. - web: page interaction, navigation, browser diagnostics. When working with a web page. - test: test analysis, state inspection. When a test is running or analyzing results. + - heal: browser/test recovery. When a test is running and browser state is broken or unavailable. ${this.idleModePrompt()} - ${mode === 'web' ? this.webModePrompt() : ''} - ${mode === 'test' ? this.testModePrompt() : ''} + ${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''} + ${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''} - After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification - Prefer completing in fewer tool calls over thoroughness - NEVER run tests unless the user explicitly asks - ${mode === 'web' ? this.webModeRules() : ''} - ${mode === 'test' ? this.testModeRules() : ''} + ${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''} + ${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''} + ${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''} ${customPrompt || ''} @@ -286,11 +293,12 @@ export class Captain extends CaptainBase implements Agent { } private async tools(task: Task, onDone: (summary: string) => void) { - const mode = this.detectMode(); + const mode = this.getMode(); const ctx: ModeContext = { explorBot: this.explorBot, task }; const core = this.coreTools(task, onDone); const idle = await this.idleModeTools(ctx); + if (mode === 'heal') return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) }; if (mode === 'test') return { ...core, ...idle, ...this.testModeTools(ctx) }; if (mode === 'web') return { ...core, ...idle, ...this.webModeTools(ctx) }; return { ...core, ...idle }; @@ -365,20 +373,50 @@ export class Captain extends CaptainBase implements Agent { return result.object; } + async processExecutionError(error: Error, activeTest: Test): Promise { + const message = error.message || String(error); + const explorer = this.explorBot.getExplorer(); + + if (!explorer.isFatalBrowserError(error)) { + return { + action: 'continue', + message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, + }; + } + + let recovered = await explorer.recoverFromBrowserError(); + if (!recovered) { + recovered = await explorer.restartBrowser(); + } + if (recovered) { + return { + action: 'continue', + recovered: true, + message: dedent` + Captain recovered the browser after a fatal page error. + Continue the test "${activeTest.scenario}" from the restored page. + The interrupted action is not evidence that the application failed. + Inspect the restored page and retry the scenario step when it is still required. + `, + }; + } + + return { + action: 'stop', + recovered: false, + message: `Captain could not recover the browser after fatal error: ${message}`, + }; + } + async handle(input: string, options: { reset?: boolean } = {}): Promise { const stateManager = this.explorBot.getExplorer().getStateManager(); const initialState = stateManager.getCurrentState(); - if (!initialState) { - tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.'); - return null; - } - const conversation = options.reset ? this.resetConversation() : this.ensureConversation(); let isDone = false; let finalSummary: string | null = null; - const startUrl = initialState.url || ''; + const startUrl = initialState?.url || ''; const task = new Task(input, startUrl); const onDone = (summary: string) => { isDone = true; @@ -421,12 +459,14 @@ export class Captain extends CaptainBase implements Agent { } const currentState = stateManager.getCurrentState(); - if (!currentState) { + if (!currentState && this.getMode() !== 'idle') { stop(); return; } - await this.reinjectContextIfNeeded(conversation, currentState); + if (currentState) { + await this.reinjectContextIfNeeded(conversation, currentState); + } if (userInput) { const newContext = await this.getPageContext(); @@ -500,3 +540,9 @@ interface SupervisorAction { action: 'inject' | 'stop' | 'pass' | 'skip'; message: string; } + +interface ExecutionRecoveryAction { + action: 'continue' | 'stop'; + message: string; + recovered?: boolean; +} diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index 215ca99..87f4419 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -1,3 +1,5 @@ +import { existsSync, readdirSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; import { tool } from 'ai'; import { createBashTool } from 'bash-tool'; import dedent from 'dedent'; @@ -75,6 +77,41 @@ export function WithIdleMode(Base: T) { return { success: true, tests: plan.tests.length }; }, }), + project: tool({ + description: dedent` + Inspect Explorbot project configuration and recent generated artifacts. + Use this before answering questions about setup, previous sessions, reports, saved plans, or output files. + `, + inputSchema: z.object({ + view: z.enum(['config', 'artifacts']).optional().describe('config shows setup summary; artifacts lists recent generated files'), + }), + execute: async ({ view }) => { + const parser = ConfigParser.getInstance(); + const config = parser.getConfig(); + const outputDir = parser.getOutputDir(); + + if (view === 'artifacts') { + return { + success: true, + outputDir, + artifacts: listRecentArtifacts(outputDir), + suggestion: 'Use bash to read a specific small report, plan, or log file when needed.', + }; + } + + return { + success: true, + configPath: parser.getConfigPath(), + baseUrl: config.playwright?.url, + browser: config.playwright?.browser, + headed: config.playwright?.show === true, + dirs: config.dirs, + agents: Object.fromEntries(Object.entries(config.ai?.agents || {}).map(([name, agentConfig]: [string, any]) => [name, { enabled: agentConfig?.enabled !== false, hasModelOverride: !!agentConfig?.model }])), + reporterEnabled: config.reporter?.enabled === true, + apiEnabled: !!config.api, + }; + }, + }), }; if (cachedBashTool) { @@ -100,6 +137,11 @@ export function WithIdleMode(Base: T) { - Use grep to search file contents + + Use project({ view: "config" }) before explaining Explorbot setup or suggesting config improvements. + Use project({ view: "artifacts" }) before answering questions about previous sessions, reports, plans, generated tests, or logs. + + When user shares credentials, selectors, or important domain info during conversation, suggest saving it to a knowledge file using bash tool. @@ -110,6 +152,42 @@ export function WithIdleMode(Base: T) { }; } +function listRecentArtifacts(outputDir: string): Array<{ path: string; size: number; modifiedAt: string }> { + const dirs = ['reports', 'plans', 'tests', 'states']; + const artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }> = []; + + for (const dir of dirs) { + if (artifacts.length >= 200) break; + const targetDir = join(outputDir, dir); + if (!existsSync(targetDir)) continue; + collectArtifacts(outputDir, targetDir, artifacts); + } + + return artifacts + .sort((a, b) => b.timestamp - a.timestamp) + .slice(0, 20) + .map(({ timestamp, ...artifact }) => artifact); +} + +function collectArtifacts(outputDir: string, targetDir: string, artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }>): void { + for (const entry of readdirSync(targetDir, { withFileTypes: true })) { + if (artifacts.length >= 200) return; + const entryPath = join(targetDir, entry.name); + if (entry.isDirectory()) { + collectArtifacts(outputDir, entryPath, artifacts); + continue; + } + + const stats = statSync(entryPath); + artifacts.push({ + path: relative(outputDir, entryPath), + size: stats.size, + modifiedAt: stats.mtime.toISOString(), + timestamp: stats.mtimeMs, + }); + } +} + export interface IdleModeMethods { idleModeTools(ctx: ModeContext): Promise>; idleModePrompt(): string; diff --git a/src/ai/captain/mixin.ts b/src/ai/captain/mixin.ts index a86afa9..342c12f 100644 --- a/src/ai/captain/mixin.ts +++ b/src/ai/captain/mixin.ts @@ -8,7 +8,7 @@ export type Constructor = new (...args: any[]) => T; export const debugLog = createDebug('explorbot:captain'); -export type CaptainMode = 'idle' | 'web' | 'test'; +export type CaptainMode = 'idle' | 'web' | 'test' | 'heal'; export interface ModeContext { explorBot: ExplorBot; diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index 151f4dc..232fc40 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -47,18 +47,53 @@ export function WithWebMode(Base: T) { description: dedent` Direct browser access via Playwright. Use for diagnostics and browser management. Actions: + - status: Inspect browser/page availability, URL, title, tab count - evaluate: Run JavaScript in browser context (localStorage, cookies, DOM, console) - closeTabs: Close all browser tabs except the current one - - screenshot: Take a screenshot of current page - reload: Reload the current page + - screenshot: Take a screenshot of current page + - recover: Recover from a closed/crashed page using Explorer recovery + - restart: Restart the browser when page/context recovery is not enough + - openFreshTab: Open a fresh tab in the current browser context `, inputSchema: z.object({ - action: z.enum(['evaluate', 'closeTabs', 'screenshot', 'reload']).describe('Browser action to perform'), + action: z.enum(['status', 'evaluate', 'closeTabs', 'reload', 'screenshot', 'recover', 'restart', 'openFreshTab']).describe('Browser action to perform'), code: z.string().optional().describe('JavaScript code for evaluate action'), }), execute: async ({ action, code }) => { - const page = ctx.explorBot.getExplorer().playwrightHelper?.page; - if (!page) return { success: false, message: 'No browser page available' }; + const explorer = ctx.explorBot.getExplorer(); + + if (action === 'status') { + const page = explorer.playwrightHelper?.page; + const pages = page?.context?.().pages?.() || []; + return { + success: true, + hasPage: !!page, + isClosed: page?.isClosed?.() || false, + url: page && !page.isClosed?.() ? await page.url() : null, + title: page && !page.isClosed?.() ? await page.title().catch(() => null) : null, + tabs: pages.length, + }; + } + + if (action === 'recover') { + const recovered = await explorer.recoverFromBrowserError(); + return { success: recovered, message: recovered ? 'Browser page recovered' : 'Browser recovery failed' }; + } + + if (action === 'restart') { + const restarted = await explorer.restartBrowser(); + return { success: restarted, message: restarted ? 'Browser restarted' : 'Browser restart failed' }; + } + + if (action === 'openFreshTab') { + await ctx.explorBot.openFreshTab(); + const state = explorer.getStateManager().getCurrentState(); + return { success: true, url: state?.url, title: state?.title }; + } + + const page = explorer.playwrightHelper?.page; + if (!page || page.isClosed?.()) return { success: false, message: 'No browser page available. Try browser({ action: "recover" }) first.' }; if (action === 'evaluate') { if (!code) return { success: false, message: 'Code required for evaluate action' }; @@ -112,7 +147,7 @@ export function WithWebMode(Base: T) { - Page actions: click, pressKey, form (CodeceptJS tools) - Navigation: navigate() — AI-powered navigation to URLs or page descriptions - - Browser diagnostics: browser() — evaluate JS, close tabs, screenshot, reload + - Browser diagnostics: browser() — inspect status, evaluate JS, close tabs, screenshot, reload, recover closed/crashed pages, restart browser, open a fresh tab - Visual analysis: see() — screenshot-based page verification - Context refresh: context() — get fresh HTML/ARIA snapshot - Visual fallback: visualClick() — coordinate-based click when locators fail diff --git a/src/ai/historian/screencast.ts b/src/ai/historian/screencast.ts index 15e03f0..b7586cb 100644 --- a/src/ai/historian/screencast.ts +++ b/src/ai/historian/screencast.ts @@ -10,6 +10,8 @@ import { relativeToCwd } from '../../utils/next-steps.ts'; import { safeFilename } from '../../utils/strings.ts'; import { type Constructor, debugLog } from './mixin.ts'; +const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i; + export interface ScreencastMethods { attachScreencast(): void; isScreencastActive(): boolean; @@ -113,17 +115,24 @@ export function WithScreencast(Base: T) { if (!this.screencastActive) return; const path = this.screencastPath; const task = this.screencastTask; + let stopped = false; try { await this.screencastPage.screencast.stop(); + stopped = true; } catch (err) { - tag('substep').log(`Screencast stop failed: ${(err as Error).message}`); + const message = (err as Error).message; + if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) { + tag('substep').log('Screencast skipped: browser was closed before recording could be finalized'); + } else { + tag('substep').log(`Screencast stop failed: ${message}`); + } } this.screencastActive = false; this.screencastPage = null; this.screencastPath = null; this.screencastTask = null; this.screencastLastChapter = null; - if (path) { + if (path && stopped) { this.savedFiles.add(path); task?.addArtifact?.(path); tag('substep').log(`Saved screencast: ${relativeToCwd(path)}`); diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 4e7163d..db6e4b4 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -67,7 +67,7 @@ export class Pilot implements Agent { } async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise { - const verdictType = task.hasAchievedAny() ? 'finish' : 'stop'; + const verdictType = task.hasAchievedAny() || this.hasSuccessfulAssertionEvidence(currentState, testerConversation) ? 'finish' : 'stop'; return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator); } @@ -86,6 +86,7 @@ export class Pilot implements Agent { const sessionLog = this.formatSessionLog(testerConversation); const stateContext = this.buildStateContext(currentState); + const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation); const notes = task.notesToString() || 'No notes recorded.'; let visualAnalysis = ''; @@ -125,6 +126,10 @@ export class Pilot implements Agent { ${this.formatExpectations(task)} + + ${successfulAssertions || 'None'} + + ${notes} @@ -880,6 +885,27 @@ export class Pilot implements Agent { return parts.join('\n\n'); } + private hasSuccessfulAssertionEvidence(currentState: ActionResult, testerConversation: Conversation): boolean { + if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true; + return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful); + } + + private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string { + const lines: string[] = []; + for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) { + if (passed) lines.push(`PASS state verification: ${assertion}`); + } + + for (const exec of testerConversation.getToolExecutions()) { + if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue; + const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input); + const result = exec.output?.message || exec.output?.analysis || exec.output?.result; + lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`); + } + + return [...new Set(lines)].join('\n'); + } + private formatActions(toolCalls: any[]): string { return toolCalls .map((t) => { diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 65961e4..b3e459a 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -137,6 +137,7 @@ export class Tester extends TaskAgent implements Agent { }); const page = this.explorer.playwrightHelper?.page; + const observedPages = new Set(); const onPageError = (err: Error) => { task.addNote(`Console error: ${err.message}`, TestResult.FAILED); }; @@ -144,8 +145,21 @@ export class Tester extends TaskAgent implements Agent { if (msg.type() !== 'error') return; task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED); }; - page?.on('pageerror', onPageError); - page?.on('console', onConsoleMessage); + const watchPage = (targetPage: any) => { + if (!targetPage) return; + if (observedPages.has(targetPage)) return; + targetPage.on('pageerror', onPageError); + targetPage.on('console', onConsoleMessage); + observedPages.add(targetPage); + }; + const unwatchPages = () => { + for (const observedPage of observedPages) { + observedPage.off('pageerror', onPageError); + observedPage.off('console', onConsoleMessage); + } + observedPages.clear(); + }; + watchPage(page); const initialState = ActionResult.fromState(state); @@ -176,20 +190,19 @@ export class Tester extends TaskAgent implements Agent { expected: task.expected, }, }, - async () => this.runTestSession(task, initialState, conversation, { offFailedRequest, page, onPageError, onConsoleMessage }) + async () => this.runTestSession(task, initialState, conversation, { offFailedRequest, watchPage, unwatchPages }) ); } - private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: { offFailedRequest?: () => void; page: any; onPageError: (err: Error) => void; onConsoleMessage: (msg: any) => void }): Promise<{ success: boolean }> { - const { offFailedRequest, page, onPageError, onConsoleMessage } = handlers; + private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers): Promise<{ success: boolean }> { + const { offFailedRequest, watchPage, unwatchPages } = handlers; if (this.pilot) { try { const plan = await this.pilot.planTest(task, initialState); if (task.hasFinished) { offFailedRequest?.(); - page?.off('pageerror', onPageError); - page?.off('console', onConsoleMessage); + unwatchPages(); return { success: task.isSuccessful }; } if (plan) { @@ -201,8 +214,7 @@ export class Tester extends TaskAgent implements Agent { task.addNote(`Planning failed: ${message}`, TestResult.FAILED); task.finish(TestResult.FAILED); offFailedRequest?.(); - page?.off('pageerror', onPageError); - page?.off('console', onConsoleMessage); + unwatchPages(); return { success: false }; } } @@ -211,9 +223,37 @@ export class Tester extends TaskAgent implements Agent { task.start(); await this.explorer.startTest(task); + let initialSetupStopped = false; + if ( + !(await this.ensureBrowserPageAvailable( + task, + conversation, + () => { + initialSetupStopped = true; + }, + watchPage + )) + ) { + offFailedRequest?.(); + unwatchPages(); + await this.cleanupStartedTest(task); + return { success: task.isSuccessful }; + } + if (initialSetupStopped || task.hasFinished) { + offFailedRequest?.(); + unwatchPages(); + await this.cleanupStartedTest(task); + return { success: task.isSuccessful }; + } debugLog(`Navigating to ${task.startUrl}`); - await this.explorer.visit(task.startUrl!); + const navigated = await this.visitStartUrlWithRecovery(task, conversation, watchPage); + if (!navigated) { + offFailedRequest?.(); + unwatchPages(); + await this.cleanupStartedTest(task); + return { success: task.isSuccessful }; + } const startState = this.explorer.getStateManager().getCurrentState(); if (startState) task.addUrlNote(startState); @@ -238,6 +278,7 @@ export class Tester extends TaskAgent implements Agent { await loop( async ({ stop, pause, iteration, userInput }) => { debugLog('iteration', iteration); + if (!(await this.ensureBrowserPageAvailable(task, conversation, stop, watchPage))) return; const currentState = this.getCurrentState(); const tools = { @@ -385,22 +426,28 @@ export class Tester extends TaskAgent implements Agent { } : undefined, catch: async ({ error, stop }) => { - tag('error').log(`Test execution error: ${error}`); - const message = error instanceof Error ? error.message : String(error); - if (!task.hasFinished) { - task.addNote(`Execution error: ${message}`); - } - if (error instanceof Error && error.name === 'AbortError') { - stop(); - return; - } - conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`); + await this.handleExecutionError(task, conversation, error, stop, watchPage); }, } ); if (task.hasFinished) break; + let finalReviewStopped = false; + if ( + !(await this.ensureBrowserPageAvailable( + task, + conversation, + () => { + finalReviewStopped = true; + }, + watchPage + )) + ) { + break; + } + if (finalReviewStopped || task.hasFinished) break; + const finalState = this.getCurrentState(); const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator); @@ -429,8 +476,7 @@ export class Tester extends TaskAgent implements Agent { offStateChange(); offFailedRequest?.(); - page?.off('pageerror', onPageError); - page?.off('console', onConsoleMessage); + unwatchPages(); await this.finishTest(task); await this.explorer.stopTest(task, { startUrl: task.startUrl, @@ -1091,4 +1137,125 @@ export class Tester extends TaskAgent implements Agent { }), }; } + + private async handleExecutionError(task: Test, conversation: Conversation, error: Error, stop: () => void, watchPage: (page: any) => void): Promise { + tag('error').log(`Test execution error: ${error}`); + const message = error instanceof Error ? error.message : String(error); + if (!task.hasFinished) { + task.addNote(`Execution error: ${message}`); + } + if (error instanceof Error && error.name === 'AbortError') { + stop(); + return; + } + if (this.captain && error instanceof Error) { + const recovery = await this.captain.processExecutionError(error, task); + tag('info').log(`Supervisor: ${recovery.action} - ${recovery.message}`); + task.addNote(recovery.message); + if (recovery.action === 'stop') { + task.finish(TestResult.FAILED); + stop(); + return; + } + if (recovery.recovered) { + watchPage(this.explorer.playwrightHelper?.page); + this.resetFailureCount(); + const recoveryContext = await this.buildRecoveryContext(task); + conversation.addUserText(`${recovery.message}\n\n${recoveryContext}`); + return; + } + conversation.addUserText(recovery.message); + return; + } + const isFatalBrowserError = this.explorer.isFatalBrowserError?.(error) ?? /Target closed|Session closed|Protocol error/i.test(message); + if (isFatalBrowserError) { + task.finish(TestResult.FAILED); + stop(); + return; + } + conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`); + } + + private async cleanupStartedTest(task: Test): Promise { + await this.finishTest(task); + await this.explorer.stopTest(task, { + startUrl: task.startUrl, + style: task.style, + sessionName: task.sessionName, + }); + } + + private async buildRecoveryContext(task: Test): Promise { + let currentState: ActionResult | null = null; + try { + currentState = await this.explorer.createAction().capturePageState(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return dedent` + + Browser was restored, but fresh page context could not be captured: ${message} + The previous Target closed error was an external browser interruption, not product evidence. + Re-open or inspect the page before deciding pass/fail. + + `; + } + + this.previousUrl = null; + this.previousStateHash = null; + const pageContext = await this.reinjectContextIfNeeded(1, currentState); + return dedent` + + Browser was restored during "${task.scenario}". + The previous Target closed error was an external browser interruption, not product evidence. + Ignore interrupted click/form attempts when judging whether the application works. + Retry the current scenario step from the restored page if the goal is still incomplete. + + + ${pageContext} + `; + } + + private async ensureBrowserPageAvailable(task: Test, conversation: Conversation, stop: () => void, watchPage: (page: any) => void): Promise { + const page = this.explorer.playwrightHelper?.page; + if (page && !page.isClosed?.()) return true; + + await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), stop, watchPage); + return !task.hasFinished; + } + + private async visitStartUrlWithRecovery(task: Test, conversation: Conversation, watchPage: (page: any) => void): Promise { + try { + await this.explorer.visit(task.startUrl!); + return true; + } catch (error) { + let stopped = false; + await this.handleExecutionError( + task, + conversation, + error instanceof Error ? error : new Error(String(error)), + () => { + stopped = true; + }, + watchPage + ); + + if (stopped || task.hasFinished) return false; + + try { + await this.explorer.visit(task.startUrl!); + return true; + } catch (retryError) { + const message = retryError instanceof Error ? retryError.message : String(retryError); + task.addNote(`Initial navigation failed after recovery: ${message}`, TestResult.FAILED); + task.finish(TestResult.FAILED); + return false; + } + } + } +} + +interface TestSessionHandlers { + offFailedRequest?: () => void; + watchPage: (page: any) => void; + unwatchPages: () => void; } diff --git a/src/explorer.ts b/src/explorer.ts index d93e045..73ead20 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -199,12 +199,14 @@ class Explorer { } } - private setupXhrCapture(): void { + private setupXhrCapture(reuseRequestStore = false): void { const configParser = ConfigParser.getInstance(); const outputDir = configParser.getOutputDir(); - this.requestStore = new RequestStore(outputDir); + if (!reuseRequestStore || !this.requestStore) { + this.requestStore = new RequestStore(outputDir); + } const baseUrl = this.config.playwright.url; - this.xhrCapture = new XhrCapture(this.requestStore, baseUrl); + this.xhrCapture = new XhrCapture(this.requestStore!, baseUrl); this.xhrCapture.attach(this.playwrightHelper.page); } @@ -239,18 +241,7 @@ class Explorer { } await this.connectOrLaunchBrowser(); const hasSession = this.options?.session && existsSync(this.options.session); - const helperOptions = this.playwrightHelper.options || {}; - // CodeceptJS skips _createContextPage when sessions/storageState are involved, so we - // build contextOptions ourselves. Most keys share a name with Playwright's - // BrowserContextOptions and are copied as-is; `emulate` must be flattened, `basicAuth` - // renamed to `httpCredentials`, and `storageState` comes from the --session flag. - const contextOptions: BrowserContextOptions = { - ...helperOptions, - }; - if (helperOptions.emulate) Object.assign(contextOptions, helperOptions.emulate); - if (helperOptions.basicAuth) contextOptions.httpCredentials = helperOptions.basicAuth; - if (hasSession) contextOptions.storageState = this.options!.session; - await this.playwrightHelper._createContextPage(contextOptions); + await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); await this.playwrightRecorder.start(this.playwrightHelper.browserContext); this.setupXhrCapture(); if (hasSession) { @@ -287,6 +278,19 @@ class Explorer { await this.playwrightHelper._startBrowser(); } + private createBrowserContextOptions(): BrowserContextOptions { + const helperOptions = this.playwrightHelper.options || {}; + const contextOptions: BrowserContextOptions = { + ...helperOptions, + }; + + if (helperOptions.emulate) Object.assign(contextOptions, helperOptions.emulate); + if (helperOptions.basicAuth) contextOptions.httpCredentials = helperOptions.basicAuth; + if (this.options?.session && existsSync(this.options.session)) contextOptions.storageState = this.options.session; + + return contextOptions; + } + createAction() { return new Action(this.actor, this.stateManager, this.playwrightRecorder); } @@ -386,6 +390,22 @@ class Explorer { await this.playwrightHelper.page.reload(); } + private resolveBrowserUrl(url?: string): string | null { + if (!url) return null; + try { + return new URL(url).toString(); + } catch {} + + const baseUrl = this.config.playwright?.url || this.config.web?.url; + if (!baseUrl) return null; + + try { + return new URL(url, baseUrl).toString(); + } catch { + return null; + } + } + isFatalBrowserError(error: unknown): boolean { const msg = error instanceof Error ? error.message : String(error); return FATAL_BROWSER_ERRORS.test(msg); @@ -393,7 +413,19 @@ class Explorer { async recoverFromBrowserError(): Promise { try { - const url = this.stateManager.getCurrentState()?.url; + if (!this.playwrightHelper?.page || this.playwrightHelper.page.isClosed?.()) { + const context = this.playwrightHelper?.browserContext; + if (!context) return await this.restartBrowser(); + const page = await context.newPage(); + await page.bringToFront(); + this.playwrightHelper.page = page; + this.bindFrameNavigated(page); + if (this.xhrCapture) { + this.xhrCapture.attach(this.playwrightHelper.page); + } + } + + const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); if (url) { tag('warning').log(`Browser error detected, recovering by navigating to ${url}`); await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); @@ -408,6 +440,49 @@ class Explorer { } } + async restartBrowser(): Promise { + if (!this.playwrightHelper) return false; + + const url = this.resolveBrowserUrl(this.stateManager.getCurrentState()?.url); + + try { + if (this.xhrCapture && this.playwrightHelper.page) { + this.xhrCapture.detach(this.playwrightHelper.page); + } + + await this.playwrightRecorder.stop(); + + if (this.playwrightHelper.browserContext) { + await this.playwrightHelper.browserContext.close().catch((err: unknown) => { + debugLog('Failed to close browser context before restart:', err); + }); + this.playwrightHelper.browserContext = null; + } + + if (!this.isSharedBrowser) { + await this.playwrightHelper._stopBrowser().catch((err: unknown) => { + debugLog('Failed to stop browser before restart:', err); + }); + } + + await this.connectOrLaunchBrowser(); + await this.playwrightHelper._createContextPage(this.createBrowserContextOptions()); + await this.playwrightRecorder.start(this.playwrightHelper.browserContext); + this.setupXhrCapture(true); + this.listenToStateChanged(); + + if (url) { + await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + } + + tag('success').log('Browser restarted'); + return true; + } catch (err) { + tag('error').log(`Browser restart failed: ${err instanceof Error ? err.message : err}`); + return false; + } + } + async switchToMainFrame() { if (this.playwrightHelper.frame) { debugLog('Switching to main frame'); @@ -705,7 +780,6 @@ class Explorer { } await firstPage.bringToFront(); - this.playwrightHelper.page = firstPage; debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); diff --git a/tests/unit/captain-mode.test.ts b/tests/unit/captain-mode.test.ts new file mode 100644 index 0000000..83dc7ab --- /dev/null +++ b/tests/unit/captain-mode.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from 'bun:test'; +import { Captain } from '../../src/ai/captain.ts'; + +function buildCaptain(opts: { state?: any; activeTest?: any; page?: any }) { + const explorer = { + activeTest: opts.activeTest || null, + playwrightHelper: { + page: opts.page, + }, + getStateManager: () => ({ + getCurrentState: () => opts.state || null, + }), + }; + + const explorBot = { + getExplorer: () => explorer, + }; + + return Object.assign(Object.create(Captain.prototype), { explorBot }) as Captain; +} + +function buildCaptainWithExplorer(explorer: any) { + return Object.assign(Object.create(Captain.prototype), { + explorBot: { + getExplorer: () => explorer, + }, + }) as Captain; +} + +describe('Captain modes', () => { + it('uses idle mode without a loaded page', () => { + const captain = buildCaptain({}); + + expect(captain.getMode()).toBe('idle'); + }); + + it('uses web mode when a page state exists', () => { + const captain = buildCaptain({ state: { url: '/dashboard' } }); + + expect(captain.getMode()).toBe('web'); + }); + + it('uses test mode while a test is active', () => { + const captain = buildCaptain({ + activeTest: { sessionName: 'test-session' }, + page: { isClosed: () => false }, + state: { url: '/dashboard' }, + }); + + expect(captain.getMode()).toBe('test'); + }); + + it('uses heal mode when active test has no usable browser page', () => { + const captain = buildCaptain({ + activeTest: { sessionName: 'test-session' }, + page: { isClosed: () => true }, + state: { url: '/dashboard' }, + }); + + expect(captain.getMode()).toBe('heal'); + }); + +}); + +describe('Captain execution recovery', () => { + it('continues after a fatal browser error is recovered', async () => { + const captain = buildCaptainWithExplorer({ + isFatalBrowserError: () => true, + recoverFromBrowserError: async () => true, + }); + + const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); + + expect(recovery.action).toBe('continue'); + expect(recovery.recovered).toBe(true); + expect(recovery.message).toContain('Captain recovered the browser'); + }); + + it('stops when a fatal browser error cannot be recovered', async () => { + const captain = buildCaptainWithExplorer({ + isFatalBrowserError: () => true, + recoverFromBrowserError: async () => false, + restartBrowser: async () => false, + }); + + const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); + + expect(recovery.action).toBe('stop'); + expect(recovery.recovered).toBe(false); + }); + + it('continues when browser restart recovers after page recovery fails', async () => { + const captain = buildCaptainWithExplorer({ + isFatalBrowserError: () => true, + recoverFromBrowserError: async () => false, + restartBrowser: async () => true, + }); + + const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); + + expect(recovery.action).toBe('continue'); + expect(recovery.recovered).toBe(true); + }); + + it('continues with guidance for non-fatal execution errors', async () => { + const captain = buildCaptainWithExplorer({ + isFatalBrowserError: () => false, + recoverFromBrowserError: async () => false, + }); + + const recovery = await captain.processExecutionError(new Error('Locator not found'), { scenario: 'create project' } as any); + + expect(recovery.action).toBe('continue'); + expect(recovery.recovered).toBeUndefined(); + expect(recovery.message).toContain('Previous execution error'); + }); +}); diff --git a/tests/unit/explorer-recovery-url.test.ts b/tests/unit/explorer-recovery-url.test.ts new file mode 100644 index 0000000..f4a1565 --- /dev/null +++ b/tests/unit/explorer-recovery-url.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'bun:test'; +import Explorer from '../../src/explorer.ts'; + +function buildExplorer(baseUrl: string) { + return Object.assign(Object.create(Explorer.prototype), { + config: { + playwright: { url: baseUrl }, + web: { url: baseUrl }, + }, + }) as Explorer; +} + +describe('Explorer recovery URL resolution', () => { + it('resolves path-only state URLs against the configured base URL', () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + + expect((explorer as any).resolveBrowserUrl('/')).toBe('https://the-internet.herokuapp.com/'); + expect((explorer as any).resolveBrowserUrl('/add_remove_elements/')).toBe('https://the-internet.herokuapp.com/add_remove_elements/'); + }); + + it('keeps absolute state URLs unchanged', () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + + expect((explorer as any).resolveBrowserUrl('https://example.test/page')).toBe('https://example.test/page'); + }); + + it('creates a fresh active page when recovering a closed page', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + const navigated: string[] = []; + const boundEvents: string[] = []; + const newPage = { + goto: async (url: string) => { + navigated.push(url); + }, + bringToFront: async () => {}, + on: (event: string) => { + boundEvents.push(event); + }, + mainFrame: () => ({}), + }; + (explorer as any).playwrightHelper = { + page: { isClosed: () => true }, + browserContext: { + newPage: async () => newPage, + }, + }; + (explorer as any).stateManager = { + getCurrentState: () => ({ url: '/' }), + updateStateFromBasic: () => {}, + }; + + const recovered = await explorer.recoverFromBrowserError(); + + expect(recovered).toBe(true); + expect((explorer as any).playwrightHelper.page).toBe(newPage); + expect(navigated).toEqual(['https://the-internet.herokuapp.com/']); + expect(boundEvents).toContain('framenavigated'); + }); + +}); diff --git a/tests/unit/historian-screencast.test.ts b/tests/unit/historian-screencast.test.ts new file mode 100644 index 0000000..3c0d5ce --- /dev/null +++ b/tests/unit/historian-screencast.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'bun:test'; +import { WithScreencast } from '../../src/ai/historian/screencast.ts'; + +function buildScreencastHost(stop: () => Promise) { + const Host = WithScreencast(Object as unknown as new () => object); + const host: any = new Host(); + host.savedFiles = new Set(); + host.screencastActive = true; + host.screencastPath = 'output/screencasts/test.webm'; + host.screencastPage = { + screencast: { stop }, + }; + const artifacts: string[] = []; + host.screencastTask = { + addArtifact: (path: string) => artifacts.push(path), + }; + return { host, artifacts }; +} + +describe('Historian screencast cleanup', () => { + it('does not save screencast artifact when browser was closed before stop', async () => { + const { host, artifacts } = buildScreencastHost(async () => { + throw new Error('stop: Target page, context or browser has been closed'); + }); + + await host.stopScreencast(); + + expect(host.savedFiles.size).toBe(0); + expect(artifacts).toHaveLength(0); + expect(host.isScreencastActive()).toBe(false); + }); + + it('saves screencast artifact after a clean stop', async () => { + const { host, artifacts } = buildScreencastHost(async () => {}); + + await host.stopScreencast(); + + expect(host.savedFiles.has('output/screencasts/test.webm')).toBe(true); + expect(artifacts).toEqual(['output/screencasts/test.webm']); + }); +}); diff --git a/tests/unit/pilot-evidence.test.ts b/tests/unit/pilot-evidence.test.ts new file mode 100644 index 0000000..f6bf60f --- /dev/null +++ b/tests/unit/pilot-evidence.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'bun:test'; +import { Pilot } from '../../src/ai/pilot.ts'; + +function buildPilot(): Pilot { + return Object.assign(Object.create(Pilot.prototype), {}) as Pilot; +} + +describe('Pilot evidence', () => { + it('treats passed state verifications as successful assertion evidence', () => { + const pilot = buildPilot(); + const state = { verifications: { 'Heading is visible': true } }; + const conversation = { getToolExecutions: () => [] }; + + expect((pilot as any).hasSuccessfulAssertionEvidence(state, conversation)).toBe(true); + expect((pilot as any).formatSuccessfulAssertions(state, conversation)).toContain('PASS state verification'); + }); + + it('treats successful check tools as assertion evidence', () => { + const pilot = buildPilot(); + const state = {}; + const conversation = { + getToolExecutions: () => [ + { + toolName: 'verify', + wasSuccessful: true, + input: { assertion: 'Heading is visible' }, + output: { message: 'Verification passed: Heading is visible' }, + }, + ], + }; + + expect((pilot as any).hasSuccessfulAssertionEvidence(state, conversation)).toBe(true); + expect((pilot as any).formatSuccessfulAssertions(state, conversation)).toContain('PASS verify'); + }); +}); diff --git a/tests/unit/tester-execution-recovery.test.ts b/tests/unit/tester-execution-recovery.test.ts new file mode 100644 index 0000000..021fffb --- /dev/null +++ b/tests/unit/tester-execution-recovery.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, it } from 'bun:test'; +import { Tester } from '../../src/ai/tester.ts'; +import { TestResult } from '../../src/test-plan.ts'; + +function buildTester(captain?: any, page: any = { id: 'recovered-page' }, explorerOverrides: Record = {}): Tester { + const explorer: any = { + getConfig: () => ({}), + playwrightHelper: { + page, + }, + createAction: () => ({ + capturePageState: async () => ({ + url: '/', + title: 'Recovered', + hash: 'recovered', + ariaSnapshot: '', + getInteractiveARIA: () => '', + isInsideIframe: false, + }), + }), + hasOtherTabs: () => false, + getCurrentIframeInfo: () => null, + stopTest: async () => {}, + ...explorerOverrides, + }; + const researcher = { + research: async () => '', + }; + const tester = new Tester(explorer, {} as any, researcher as any, {} as any); + if (captain) tester.setCaptain(captain); + return tester; +} + +function buildTask() { + const notes: string[] = []; + return { + hasFinished: false, + result: null, + addNote: (message: string) => notes.push(message), + finish(result: any) { + this.hasFinished = true; + this.result = result; + }, + get isSuccessful() { + return this.result === TestResult.PASSED; + }, + get isSkipped() { + return this.result === TestResult.SKIPPED; + }, + get hasFailed() { + return this.result === TestResult.FAILED; + }, + notes, + scenario: 'startup recovery test', + }; +} + +function buildConversation() { + const messages: string[] = []; + return { + addUserText: (message: string) => messages.push(message), + messages, + }; +} + +describe('Tester execution recovery', () => { + it('continues after Captain recovers the browser', async () => { + const captain = { + processExecutionError: async () => ({ + action: 'continue', + recovered: true, + message: 'Recovered browser, continue from restored page', + }), + }; + const recoveredPage = { id: 'recovered-page' }; + const tester = buildTester(captain, { isClosed: () => true }); + (tester as any).explorer.playwrightHelper.page = recoveredPage; + const task = buildTask(); + const conversation = buildConversation(); + const watchedPages: any[] = []; + let stopped = false; + + await (tester as any).handleExecutionError( + task, + conversation, + new Error('Target closed'), + () => { + stopped = true; + }, + (page: any) => watchedPages.push(page) + ); + + expect(stopped).toBe(false); + expect(task.hasFinished).toBe(false); + expect(watchedPages).toHaveLength(1); + expect(conversation.messages[0]).toContain('Recovered browser'); + expect(conversation.messages[0]).toContain(''); + }); + + it('stops the test when Captain cannot recover', async () => { + const captain = { + processExecutionError: async () => ({ + action: 'stop', + recovered: false, + message: 'Recovery failed', + }), + }; + const tester = buildTester(captain); + const task = buildTask(); + const conversation = buildConversation(); + let stopped = false; + + await (tester as any).handleExecutionError( + task, + conversation, + new Error('Target closed'), + () => { + stopped = true; + }, + () => {} + ); + + expect(stopped).toBe(true); + expect(task.hasFinished).toBe(true); + expect(task.result).toBe(TestResult.FAILED); + expect(conversation.messages).toHaveLength(0); + }); + + it('falls back to retry guidance when Captain is unavailable', async () => { + const tester = buildTester(); + const task = buildTask(); + const conversation = buildConversation(); + let stopped = false; + + await (tester as any).handleExecutionError( + task, + conversation, + new Error('Locator not found'), + () => { + stopped = true; + }, + () => {} + ); + + expect(stopped).toBe(false); + expect(conversation.messages[0]).toContain('Previous AI call failed'); + }); + + it('recovers when the browser page is already closed before the next step', async () => { + const tester = buildTester(undefined, { isClosed: () => true }); + const recoveredPage = { id: 'recovered-page' }; + const captain = { + processExecutionError: async () => { + (tester as any).explorer.playwrightHelper.page = recoveredPage; + return { + action: 'continue', + recovered: true, + message: 'Recovered closed page', + }; + }, + }; + tester.setCaptain(captain as any); + const task = buildTask(); + const conversation = buildConversation(); + const watchedPages: any[] = []; + let stopped = false; + + const available = await (tester as any).ensureBrowserPageAvailable( + task, + conversation, + () => { + stopped = true; + }, + (page: any) => watchedPages.push(page) + ); + + expect(available).toBe(true); + expect(stopped).toBe(false); + expect(watchedPages).toHaveLength(1); + expect(conversation.messages[0]).toContain('Recovered closed page'); + expect(conversation.messages[0]).toContain(''); + }); + + it('retries initial navigation after Captain recovers the browser', async () => { + let visits = 0; + const tester = buildTester(undefined, { isClosed: () => false }, { + visit: async () => { + visits++; + if (visits === 1) throw new Error('Cannot navigate: page has been closed'); + }, + isFatalBrowserError: () => true, + }); + const captain = { + processExecutionError: async () => ({ + action: 'continue', + recovered: true, + message: 'Recovered before initial navigation', + }), + }; + tester.setCaptain(captain as any); + const task = buildTask(); + task.startUrl = '/'; + const conversation = buildConversation(); + const watchedPages: any[] = []; + + const navigated = await (tester as any).visitStartUrlWithRecovery(task, conversation, (page: any) => watchedPages.push(page)); + + expect(navigated).toBe(true); + expect(visits).toBe(2); + expect(task.hasFinished).toBe(false); + expect(watchedPages).toHaveLength(1); + }); + + it('cleans up started test lifecycle on early startup failure', async () => { + let stopped = false; + const tester = buildTester(undefined, { isClosed: () => false }, { + stopTest: async () => { + stopped = true; + }, + }); + const task = buildTask(); + task.startUrl = '/'; + + await (tester as any).cleanupStartedTest(task); + + expect(stopped).toBe(true); + }); +}); From 3d2d77908ae99f2a1ae609804bf083c2310d0c09 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Wed, 3 Jun 2026 21:39:47 +0300 Subject: [PATCH 2/6] upd fixer captain and fix formatter --- src/ai/captain.ts | 23 ++++++ src/ai/captain/idle-mode.ts | 56 ++++++++++++- tests/unit/captain-artifacts.test.ts | 85 ++++++++++++++++++++ tests/unit/captain-mode.test.ts | 1 - tests/unit/explorer-recovery-url.test.ts | 1 - tests/unit/tester-execution-recovery.test.ts | 32 +++++--- 6 files changed, 182 insertions(+), 16 deletions(-) create mode 100644 tests/unit/captain-artifacts.test.ts diff --git a/src/ai/captain.ts b/src/ai/captain.ts index 1eb471e..2066905 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -268,6 +268,9 @@ export class Captain extends CaptainBase implements Agent { runCommand: tool({ description: dedent` Execute a TUI command. Returns log output from command execution. + Use only when the user explicitly asks to run a slash command. + Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience. + Never run /test, /rerun, /plan, /plans, /plan:load, /experience, /learn, or /drill unless the user explicitly requested that slash command. ${this.commandDescriptions .map((c) => { const opts = c.options ? ` (${c.options})` : ''; @@ -281,6 +284,13 @@ export class Captain extends CaptainBase implements Agent { execute: async ({ command }) => { if (!this.commandExecutor) return { success: false, message: 'Command executor not available' }; const cmd = command.startsWith('/') ? command : `/${command}`; + if (isUnsafeImplicitCommand(cmd) && !isExplicitSlashRequest(task.description, cmd)) { + return { + success: false, + command: cmd, + message: 'Command blocked: slash commands that can run tests, mutate plans, or inspect unrelated project sections require an explicit user slash-command request. Use readArtifact/project/test inspection tools for analysis instead.', + }; + } startLogCapture(); try { await this.commandExecutor(cmd); @@ -546,3 +556,16 @@ interface ExecutionRecoveryAction { message: string; recovered?: boolean; } + +function isUnsafeImplicitCommand(command: string): boolean { + return /^\/(?:test|rerun|plan(?::load)?|plans|experience|learn|drill)(?:\s|$)/.test(command.trim()); +} + +function isExplicitSlashRequest(input: string, command: string): boolean { + const normalizedInput = input.trim(); + if (!normalizedInput.startsWith('/')) return false; + + const requested = normalizedInput.split(/\s+/)[0]; + const actual = command.trim().split(/\s+/)[0]; + return requested === actual; +} diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index 87f4419..e238874 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -1,5 +1,5 @@ -import { existsSync, readdirSync, statSync } from 'node:fs'; -import { join, relative } from 'node:path'; +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { basename, isAbsolute, join, relative, resolve } from 'node:path'; import { tool } from 'ai'; import { createBashTool } from 'bash-tool'; import dedent from 'dedent'; @@ -112,6 +112,38 @@ export function WithIdleMode(Base: T) { }; }, }), + readArtifact: tool({ + description: dedent` + Read a specific small Explorbot artifact file for analysis. + Use this for explicit user questions about reports, plans, logs, generated tests, knowledge, or experience files. + Prefer this over runCommand() for file analysis. + `, + inputSchema: z.object({ + path: z.string().describe('Artifact path, such as output/reports/session-name-tests.md or explorbot-testing/output/reports/session-name-tests.md'), + maxChars: z.number().optional().describe('Maximum characters to return, default 12000'), + }), + execute: async ({ path, maxChars }) => { + const resolved = resolveReadableArtifact(projectRoot, path); + if (!resolved) { + return { success: false, message: 'File is outside allowed artifact directories' }; + } + if (!existsSync(resolved)) { + return { success: false, message: `File not found: ${path}` }; + } + if (!statSync(resolved).isFile()) { + return { success: false, message: `Not a file: ${path}` }; + } + + const limit = Math.max(1000, Math.min(maxChars || 12000, 50000)); + const content = readFileSync(resolved, 'utf8'); + return { + success: true, + path: relative(projectRoot || process.cwd(), resolved), + truncated: content.length > limit, + content: content.slice(0, limit), + }; + }, + }), }; if (cachedBashTool) { @@ -129,6 +161,7 @@ export function WithIdleMode(Base: T) { return dedent` - Plan management: updatePlan() — replace or append tests in the current plan + - readArtifact() — read a specific report, plan, log, generated test, knowledge, or experience file for analysis - bash() — run shell commands for file operations - READ from: ${knowledgeDir}/, ${experienceDir}/, output/ - WRITE to: ${knowledgeDir}/, ${experienceDir}/ only (NOT output/) @@ -188,6 +221,25 @@ function collectArtifacts(outputDir: string, targetDir: string, artifacts: Array } } +function resolveReadableArtifact(projectRoot: string | null, requestedPath: string): string | null { + if (!projectRoot) return null; + + let cleanPath = requestedPath.trim(); + const projectName = basename(projectRoot); + if (cleanPath.startsWith(`${projectName}/`) || cleanPath.startsWith(`${projectName}\\`)) { + cleanPath = cleanPath.slice(projectName.length + 1); + } + + const resolved = isAbsolute(cleanPath) ? resolve(cleanPath) : resolve(projectRoot, cleanPath); + const allowedRoots = ['output', 'knowledge', 'experience'].map((dir) => resolve(projectRoot, dir)); + for (const root of allowedRoots) { + const rel = relative(root, resolved); + if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) return resolved; + } + + return null; +} + export interface IdleModeMethods { idleModeTools(ctx: ModeContext): Promise>; idleModePrompt(): string; diff --git a/tests/unit/captain-artifacts.test.ts b/tests/unit/captain-artifacts.test.ts new file mode 100644 index 0000000..a3aabc6 --- /dev/null +++ b/tests/unit/captain-artifacts.test.ts @@ -0,0 +1,85 @@ +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { basename, dirname, join } from 'node:path'; +import { describe, expect, it } from 'bun:test'; +import { Captain } from '../../src/ai/captain.ts'; +import { ConfigParser } from '../../src/config.ts'; +import type { Task } from '../../src/test-plan.ts'; + +function buildCaptain(commandExecutor?: (cmd: string) => Promise) { + return Object.assign(Object.create(Captain.prototype), { + commandExecutor, + commandDescriptions: [], + }) as Captain; +} + +function task(description: string) { + return { description } as Task; +} + +describe('Captain artifact analysis tools', () => { + it('reads explicit report artifact paths without shell commands', async () => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + const parser = ConfigParser.getInstance(); + const outputDir = join(dirname(parser.getConfigPath()!), 'output'); + const reportDir = join(outputDir, 'reports'); + mkdirSync(reportDir, { recursive: true }); + writeFileSync(join(reportDir, 'session-demo-tests.md'), '# Failed run\n\nExpected button was missing.'); + + const captain = buildCaptain(); + const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); + const result = await tools.readArtifact.execute({ path: 'output/reports/session-demo-tests.md' }); + + expect(result.success).toBe(true); + expect(result.content).toContain('Expected button was missing'); + + rmSync(join(outputDir, '..'), { recursive: true, force: true }); + }); + + it('accepts paths prefixed with the project directory name', async () => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + const parser = ConfigParser.getInstance(); + const outputDir = join(dirname(parser.getConfigPath()!), 'output'); + const reportDir = join(outputDir, 'reports'); + mkdirSync(reportDir, { recursive: true }); + writeFileSync(join(reportDir, 'session-demo-tests.md'), '# Failed run\n\nWrong expectation.'); + + const captain = buildCaptain(); + const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); + const projectName = basename(dirname(parser.getConfigPath()!)); + const result = await tools.readArtifact.execute({ path: `${projectName}/output/reports/session-demo-tests.md` }); + + expect(result.success).toBe(true); + expect(result.content).toContain('Wrong expectation'); + + rmSync(join(outputDir, '..'), { recursive: true, force: true }); + }); +}); + +describe('Captain command guard', () => { + it('blocks test execution commands for natural-language analysis requests', async () => { + let called = false; + const captain = buildCaptain(async () => { + called = true; + }); + const tools = (captain as any).coreTools(task('analyze the latest report'), () => {}); + const result = await tools.runCommand.execute({ command: '/test failing_demo_for_captain_tui_explanation' }); + + expect(result.success).toBe(false); + expect(called).toBe(false); + expect(result.message).toContain('Command blocked'); + }); + + it('allows execution commands when the user explicitly typed that slash command', async () => { + let called = false; + const captain = buildCaptain(async () => { + called = true; + }); + const tools = (captain as any).coreTools(task('/test 1'), () => {}); + const result = await tools.runCommand.execute({ command: '/test 1' }); + + expect(result.success).toBe(true); + expect(called).toBe(true); + }); +}); diff --git a/tests/unit/captain-mode.test.ts b/tests/unit/captain-mode.test.ts index 83dc7ab..45a45cf 100644 --- a/tests/unit/captain-mode.test.ts +++ b/tests/unit/captain-mode.test.ts @@ -59,7 +59,6 @@ describe('Captain modes', () => { expect(captain.getMode()).toBe('heal'); }); - }); describe('Captain execution recovery', () => { diff --git a/tests/unit/explorer-recovery-url.test.ts b/tests/unit/explorer-recovery-url.test.ts index f4a1565..f0b7002 100644 --- a/tests/unit/explorer-recovery-url.test.ts +++ b/tests/unit/explorer-recovery-url.test.ts @@ -56,5 +56,4 @@ describe('Explorer recovery URL resolution', () => { expect(navigated).toEqual(['https://the-internet.herokuapp.com/']); expect(boundEvents).toContain('framenavigated'); }); - }); diff --git a/tests/unit/tester-execution-recovery.test.ts b/tests/unit/tester-execution-recovery.test.ts index 021fffb..dde11a5 100644 --- a/tests/unit/tester-execution-recovery.test.ts +++ b/tests/unit/tester-execution-recovery.test.ts @@ -183,13 +183,17 @@ describe('Tester execution recovery', () => { it('retries initial navigation after Captain recovers the browser', async () => { let visits = 0; - const tester = buildTester(undefined, { isClosed: () => false }, { - visit: async () => { - visits++; - if (visits === 1) throw new Error('Cannot navigate: page has been closed'); - }, - isFatalBrowserError: () => true, - }); + const tester = buildTester( + undefined, + { isClosed: () => false }, + { + visit: async () => { + visits++; + if (visits === 1) throw new Error('Cannot navigate: page has been closed'); + }, + isFatalBrowserError: () => true, + } + ); const captain = { processExecutionError: async () => ({ action: 'continue', @@ -213,11 +217,15 @@ describe('Tester execution recovery', () => { it('cleans up started test lifecycle on early startup failure', async () => { let stopped = false; - const tester = buildTester(undefined, { isClosed: () => false }, { - stopTest: async () => { - stopped = true; - }, - }); + const tester = buildTester( + undefined, + { isClosed: () => false }, + { + stopTest: async () => { + stopped = true; + }, + } + ); const task = buildTask(); task.startUrl = '/'; From 3d1c77776ecc563affeb640641c61044f5fdf6b2 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 4 Jun 2026 22:12:34 +0300 Subject: [PATCH 3/6] fix --- src/ai/captain.ts | 18 +++++++++++++++++- src/components/LogPane.tsx | 7 ++++--- src/utils/logger.ts | 6 +++--- tests/unit/captain-artifacts.test.ts | 28 ++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/ai/captain.ts b/src/ai/captain.ts index 2066905..d29c56e 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -117,6 +117,7 @@ export class Captain extends CaptainBase implements Agent { - After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification - Prefer completing in fewer tool calls over thoroughness - NEVER run tests unless the user explicitly asks + - If the user asks to show, display, explain, summarize, compare, or diagnose information, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained. ${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''} ${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''} ${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''} @@ -257,9 +258,20 @@ export class Captain extends CaptainBase implements Agent { description: 'Call when the user request is fulfilled.', inputSchema: z.object({ summary: z.string().describe('What was done'), + details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'), }), - execute: async ({ summary }) => { + execute: async ({ summary, details }) => { debugLog('done', summary); + if (requiresUserFacingDetails(task.description) && !details?.trim()) { + return { + success: false, + message: 'The user asked for information to be shown or explained. Call done() again with the actual answer in details, not only a completion summary.', + }; + } + if (details?.trim()) { + tag('details').log(details); + task.addNote(details); + } task.addNote(summary); onDone(summary); return { success: true, summary }; @@ -569,3 +581,7 @@ function isExplicitSlashRequest(input: string, command: string): boolean { const actual = command.trim().split(/\s+/)[0]; return requested === actual; } + +function requiresUserFacingDetails(input: string): boolean { + return /\b(show|display|explain|describe|summarize|analyse|analyze|compare|tell me|details|status|what|which|why|how|где|покажи|объясни|обьясни|расскажи|что|какой|какие|почему|как)\b/i.test(input); +} diff --git a/src/components/LogPane.tsx b/src/components/LogPane.tsx index b115453..7ad3ea1 100644 --- a/src/components/LogPane.tsx +++ b/src/components/LogPane.tsx @@ -120,6 +120,7 @@ const LogPane: React.FC = React.memo(({ verboseMode }) => { case 'step': return { color: 'cyan' as const, dimColor: true }; case 'multiline': + case 'details': return { color: 'gray' as const, dimColor: true }; case 'html': return { color: 'gray' as const }; @@ -142,15 +143,15 @@ const LogPane: React.FC = React.memo(({ verboseMode }) => { } const styles = getLogStyles(log.type); - if (log.type === 'multiline') { + if (log.type === 'multiline' || log.type === 'details') { const cleaned = stripAnsi(dedent(log.content)); const parsed = parseMarkdownToTerminal(cleaned); const lines = parsed.split('\n'); - const truncated = lines.length > MAX_MULTILINE_LINES ? `${lines.slice(0, MAX_MULTILINE_LINES).join('\n')}\n... (${lines.length - MAX_MULTILINE_LINES} more lines)` : parsed; + const content = log.type === 'details' ? parsed : lines.length > MAX_MULTILINE_LINES ? `${lines.slice(0, MAX_MULTILINE_LINES).join('\n')}\n... (${lines.length - MAX_MULTILINE_LINES} more lines)` : parsed; return ( - {truncated} + {content} ); diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 9765b98..adcae6b 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -10,7 +10,7 @@ import { ConfigParser } from '../config.js'; import { Observability } from '../observability.ts'; import { parseMarkdownToTerminal } from './markdown-terminal.ts'; -export type LogType = 'info' | 'success' | 'error' | 'warning' | 'debug' | 'substep' | 'step' | 'multiline' | 'html' | 'input'; +export type LogType = 'info' | 'success' | 'error' | 'warning' | 'debug' | 'substep' | 'step' | 'multiline' | 'details' | 'html' | 'input'; export interface TaggedLogEntry { type: LogType; @@ -94,7 +94,7 @@ class ConsoleDestination implements LogDestination { if (entry.type === 'debug') return; if (entry.type === 'html') return; let content = entry.content; - if (entry.type === 'multiline') { + if (entry.type === 'multiline' || entry.type === 'details') { const cleaned = stripAnsi(dedent(entry.content)); const parsed = parseMarkdownToTerminal(cleaned); content = parsed; @@ -306,7 +306,7 @@ class CaptainDestination implements LogDestination { stopCapture(): string[] { this.capturing = false; - const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline').map((e) => `[${e.type}] ${e.content}`); + const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline' && e.type !== 'details').map((e) => `[${e.type}] ${e.content}`); this.entries = []; return logs; } diff --git a/tests/unit/captain-artifacts.test.ts b/tests/unit/captain-artifacts.test.ts index a3aabc6..d7eaa21 100644 --- a/tests/unit/captain-artifacts.test.ts +++ b/tests/unit/captain-artifacts.test.ts @@ -12,11 +12,35 @@ function buildCaptain(commandExecutor?: (cmd: string) => Promise) { }) as Captain; } -function task(description: string) { - return { description } as Task; +function task(description: string, notes: string[] = []) { + return { + description, + addNote: (note: string) => notes.push(note), + } as unknown as Task; } describe('Captain artifact analysis tools', () => { + it('keeps done details as the user-facing answer', async () => { + const notes: string[] = []; + const captain = buildCaptain(); + const tools = (captain as any).coreTools(task('show config', notes), () => {}); + + const result = await tools.done.execute({ summary: 'Displayed config details', details: 'baseUrl: https://example.test\nbrowser: chromium' }); + + expect(result.success).toBe(true); + expect(notes).toEqual(['baseUrl: https://example.test\nbrowser: chromium', 'Displayed config details']); + }); + + it('rejects informational requests completed without details', async () => { + const captain = buildCaptain(); + const tools = (captain as any).coreTools(task('explain what page I am on'), () => {}); + + const result = await tools.done.execute({ summary: 'Explained current page' }); + + expect(result.success).toBe(false); + expect(result.message).toContain('actual answer in details'); + }); + it('reads explicit report artifact paths without shell commands', async () => { ConfigParser.resetForTesting(); ConfigParser.setupTestConfig(); From b0f1c814a4c4b09474e7bb4842e0494bd4943e36 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Fri, 5 Jun 2026 21:55:22 +0300 Subject: [PATCH 4/6] improve captain logic --- src/ai/captain.ts | 2 +- src/ai/captain/file-tools.ts | 126 +++++++++++++++++++ src/ai/captain/idle-mode.ts | 116 ++++------------- src/ai/pilot.ts | 9 +- src/ai/tester.ts | 124 ++++++------------ src/explorer.ts | 70 +++++++++-- tests/unit/captain-artifacts.test.ts | 64 +++++++++- tests/unit/pilot-evidence.test.ts | 13 +- tests/unit/tester-execution-recovery.test.ts | 55 +++----- 9 files changed, 346 insertions(+), 233 deletions(-) create mode 100644 src/ai/captain/file-tools.ts diff --git a/src/ai/captain.ts b/src/ai/captain.ts index d29c56e..bb7be65 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -300,7 +300,7 @@ export class Captain extends CaptainBase implements Agent { return { success: false, command: cmd, - message: 'Command blocked: slash commands that can run tests, mutate plans, or inspect unrelated project sections require an explicit user slash-command request. Use readArtifact/project/test inspection tools for analysis instead.', + message: 'Command blocked: slash commands that can run tests, mutate plans, or inspect unrelated project sections require an explicit user slash-command request. Use readFile/project/test inspection tools for analysis instead.', }; } startLogCapture(); diff --git a/src/ai/captain/file-tools.ts b/src/ai/captain/file-tools.ts new file mode 100644 index 0000000..89f6a8e --- /dev/null +++ b/src/ai/captain/file-tools.ts @@ -0,0 +1,126 @@ +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { basename, isAbsolute, join, relative, resolve } from 'node:path'; + +export const CAPTAIN_ARTIFACT_DIRS = ['reports', 'plans', 'tests', 'states'] as const; +export const CAPTAIN_ALLOWED_READ_DIRS = ['output', 'knowledge', 'experience'] as const; +export const CAPTAIN_ARTIFACT_SCAN_LIMIT = 200; +export const CAPTAIN_ARTIFACT_LIST_LIMIT = 20; +export const CAPTAIN_READ_FILE_DEFAULT_LIMIT = 12000; +export const CAPTAIN_READ_FILE_MAX_LIMIT = 50000; +export const CAPTAIN_READ_FILE_MIN_LIMIT = 1000; + +export function listRecentArtifacts(outputDir: string): Array<{ path: string; size: number; modifiedAt: string }> { + const artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }> = []; + + for (const dir of CAPTAIN_ARTIFACT_DIRS) { + if (artifacts.length >= CAPTAIN_ARTIFACT_SCAN_LIMIT) break; + const targetDir = join(outputDir, dir); + if (!existsSync(targetDir)) continue; + collectArtifacts(outputDir, targetDir, artifacts); + } + + return artifacts + .sort((a, b) => b.timestamp - a.timestamp) + .slice(0, CAPTAIN_ARTIFACT_LIST_LIMIT) + .map(({ timestamp, ...artifact }) => artifact); +} + +export function readCaptainFile(projectRoot: string | null, input: ReadCaptainFileInput, allowedDirs: readonly string[] = CAPTAIN_ALLOWED_READ_DIRS): ReadCaptainFileResult { + const resolved = resolveReadableFile(projectRoot, input.path, allowedDirs); + if (!resolved) { + return { success: false, message: 'File is outside allowed directories' }; + } + if (!existsSync(resolved)) { + return { success: false, message: `File not found: ${input.path}` }; + } + if (!statSync(resolved).isFile()) { + return { success: false, message: `Not a file: ${input.path}` }; + } + + const maxChars = normalizeMaxChars(input.maxChars); + const fullContent = readFileSync(resolved, 'utf8'); + const content = selectContent(fullContent, input); + return { + success: true, + path: relative(projectRoot || process.cwd(), resolved), + truncated: content.length > maxChars, + content: content.slice(0, maxChars), + }; +} + +function collectArtifacts(outputDir: string, targetDir: string, artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }>): void { + for (const entry of readdirSync(targetDir, { withFileTypes: true })) { + if (artifacts.length >= CAPTAIN_ARTIFACT_SCAN_LIMIT) return; + const entryPath = join(targetDir, entry.name); + if (entry.isDirectory()) { + collectArtifacts(outputDir, entryPath, artifacts); + continue; + } + + const stats = statSync(entryPath); + artifacts.push({ + path: relative(outputDir, entryPath), + size: stats.size, + modifiedAt: stats.mtime.toISOString(), + timestamp: stats.mtimeMs, + }); + } +} + +function resolveReadableFile(projectRoot: string | null, requestedPath: string, allowedDirs: readonly string[]): string | null { + if (!projectRoot) return null; + + let cleanPath = requestedPath.trim(); + const projectName = basename(projectRoot); + if (cleanPath.startsWith(`${projectName}/`) || cleanPath.startsWith(`${projectName}\\`)) { + cleanPath = cleanPath.slice(projectName.length + 1); + } + + const resolved = isAbsolute(cleanPath) ? resolve(cleanPath) : resolve(projectRoot, cleanPath); + const allowedRoots = allowedDirs.map((dir) => resolve(projectRoot, dir)); + for (const root of allowedRoots) { + const rel = relative(root, resolved); + if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) return resolved; + } + + return null; +} + +function selectContent(content: string, input: ReadCaptainFileInput): string { + if (!input.startLine && !input.endLine) return content; + + const lines = content.split(/\r?\n/); + const startIndex = resolveLineIndex(input.startLine, lines.length, 1); + const endIndex = resolveLineIndex(input.endLine, lines.length, lines.length); + if (endIndex < startIndex) return ''; + return lines.slice(startIndex - 1, endIndex).join('\n'); +} + +function resolveLineIndex(line: number | undefined, totalLines: number, fallback: number): number { + if (!line) return fallback; + if (line < 0) return Math.max(1, totalLines + line + 1); + return Math.min(Math.max(1, line), totalLines); +} + +function normalizeMaxChars(maxChars?: number): number { + return Math.max(CAPTAIN_READ_FILE_MIN_LIMIT, Math.min(maxChars || CAPTAIN_READ_FILE_DEFAULT_LIMIT, CAPTAIN_READ_FILE_MAX_LIMIT)); +} + +export interface ReadCaptainFileInput { + path: string; + startLine?: number; + endLine?: number; + maxChars?: number; +} + +export type ReadCaptainFileResult = + | { + success: true; + path: string; + truncated: boolean; + content: string; + } + | { + success: false; + message: string; + }; diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index e238874..a49abb2 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -1,11 +1,10 @@ -import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; -import { basename, isAbsolute, join, relative, resolve } from 'node:path'; import { tool } from 'ai'; import { createBashTool } from 'bash-tool'; import dedent from 'dedent'; import { z } from 'zod'; import { ConfigParser } from '../../config.ts'; import { Test } from '../../test-plan.ts'; +import { listRecentArtifacts, readCaptainFile } from './file-tools.ts'; import { type Constructor, type ModeContext, resolveProjectRoot } from './mixin.ts'; let cachedBashTool: Awaited> | null = null; @@ -17,6 +16,8 @@ export function WithIdleMode(Base: T) { const config = ConfigParser.getInstance().getConfig(); const knowledgeDir = config.dirs?.knowledge || 'knowledge'; const experienceDir = config.dirs?.experience || 'experience'; + const outputDir = config.dirs?.output || 'output'; + const readableDirs = [outputDir, knowledgeDir, experienceDir]; if (!cachedBashTool && projectRoot) { cachedBashTool = await createBashTool({ @@ -95,7 +96,7 @@ export function WithIdleMode(Base: T) { success: true, outputDir, artifacts: listRecentArtifacts(outputDir), - suggestion: 'Use bash to read a specific small report, plan, or log file when needed.', + suggestion: 'Use readFile to inspect specific reports, plans, logs, generated tests, knowledge, or experience files.', }; } @@ -112,37 +113,19 @@ export function WithIdleMode(Base: T) { }; }, }), - readArtifact: tool({ + readFile: tool({ description: dedent` - Read a specific small Explorbot artifact file for analysis. + Read a specific Explorbot project file for analysis. Use this for explicit user questions about reports, plans, logs, generated tests, knowledge, or experience files. - Prefer this over runCommand() for file analysis. + Prefer this over bash() for reading file contents after bash has found the file. `, inputSchema: z.object({ - path: z.string().describe('Artifact path, such as output/reports/session-name-tests.md or explorbot-testing/output/reports/session-name-tests.md'), + path: z.string().describe('Path inside output, knowledge, or experience directories'), + startLine: z.number().optional().describe('First line to read, 1-based. Negative values count from the end of the file'), + endLine: z.number().optional().describe('Last line to read, 1-based and inclusive. Negative values count from the end of the file'), maxChars: z.number().optional().describe('Maximum characters to return, default 12000'), }), - execute: async ({ path, maxChars }) => { - const resolved = resolveReadableArtifact(projectRoot, path); - if (!resolved) { - return { success: false, message: 'File is outside allowed artifact directories' }; - } - if (!existsSync(resolved)) { - return { success: false, message: `File not found: ${path}` }; - } - if (!statSync(resolved).isFile()) { - return { success: false, message: `Not a file: ${path}` }; - } - - const limit = Math.max(1000, Math.min(maxChars || 12000, 50000)); - const content = readFileSync(resolved, 'utf8'); - return { - success: true, - path: relative(projectRoot || process.cwd(), resolved), - truncated: content.length > limit, - content: content.slice(0, limit), - }; - }, + execute: async (input) => readCaptainFile(projectRoot, input, readableDirs), }), }; @@ -157,19 +140,27 @@ export function WithIdleMode(Base: T) { const config = ConfigParser.getInstance().getConfig(); const knowledgeDir = config.dirs?.knowledge || 'knowledge'; const experienceDir = config.dirs?.experience || 'experience'; + const outputDir = config.dirs?.output || 'output'; return dedent` - Plan management: updatePlan() — replace or append tests in the current plan - - readArtifact() — read a specific report, plan, log, generated test, knowledge, or experience file for analysis - - bash() — run shell commands for file operations - - READ from: ${knowledgeDir}/, ${experienceDir}/, output/ - - WRITE to: ${knowledgeDir}/, ${experienceDir}/ only (NOT output/) - - Use ls to list files, cat to read small files - - Use head/tail for large files to avoid excessive output - - Use grep to search file contents + - readFile() — read specific report, plan, log, generated test, knowledge, or experience file content + - bash() — discover files and inspect file metadata + - READ from: ${knowledgeDir}/, ${experienceDir}/, ${outputDir}/ + - WRITE to: ${knowledgeDir}/, ${experienceDir}/ only (NOT ${outputDir}/) + - Use wc -l -c file.txt to inspect size + - Use file file.txt to inspect type + - Use find . -name "*.md" to discover files + - Use grep -n "keyword" file.txt to find matching lines + - Use ls -lh to list files + + Use bash() for file discovery and search. Once the needed file and line range are known, + use readFile() to read its contents. Do not use bash() to print file contents. + + Use project({ view: "config" }) before explaining Explorbot setup or suggesting config improvements. Use project({ view: "artifacts" }) before answering questions about previous sessions, reports, plans, generated tests, or logs. @@ -185,61 +176,6 @@ export function WithIdleMode(Base: T) { }; } -function listRecentArtifacts(outputDir: string): Array<{ path: string; size: number; modifiedAt: string }> { - const dirs = ['reports', 'plans', 'tests', 'states']; - const artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }> = []; - - for (const dir of dirs) { - if (artifacts.length >= 200) break; - const targetDir = join(outputDir, dir); - if (!existsSync(targetDir)) continue; - collectArtifacts(outputDir, targetDir, artifacts); - } - - return artifacts - .sort((a, b) => b.timestamp - a.timestamp) - .slice(0, 20) - .map(({ timestamp, ...artifact }) => artifact); -} - -function collectArtifacts(outputDir: string, targetDir: string, artifacts: Array<{ path: string; size: number; modifiedAt: string; timestamp: number }>): void { - for (const entry of readdirSync(targetDir, { withFileTypes: true })) { - if (artifacts.length >= 200) return; - const entryPath = join(targetDir, entry.name); - if (entry.isDirectory()) { - collectArtifacts(outputDir, entryPath, artifacts); - continue; - } - - const stats = statSync(entryPath); - artifacts.push({ - path: relative(outputDir, entryPath), - size: stats.size, - modifiedAt: stats.mtime.toISOString(), - timestamp: stats.mtimeMs, - }); - } -} - -function resolveReadableArtifact(projectRoot: string | null, requestedPath: string): string | null { - if (!projectRoot) return null; - - let cleanPath = requestedPath.trim(); - const projectName = basename(projectRoot); - if (cleanPath.startsWith(`${projectName}/`) || cleanPath.startsWith(`${projectName}\\`)) { - cleanPath = cleanPath.slice(projectName.length + 1); - } - - const resolved = isAbsolute(cleanPath) ? resolve(cleanPath) : resolve(projectRoot, cleanPath); - const allowedRoots = ['output', 'knowledge', 'experience'].map((dir) => resolve(projectRoot, dir)); - for (const root of allowedRoots) { - const rel = relative(root, resolved); - if (!rel || (!rel.startsWith('..') && !isAbsolute(rel))) return resolved; - } - - return null; -} - export interface IdleModeMethods { idleModeTools(ctx: ModeContext): Promise>; idleModePrompt(): string; diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index db6e4b4..128f1b0 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -67,7 +67,7 @@ export class Pilot implements Agent { } async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise { - const verdictType = task.hasAchievedAny() || this.hasSuccessfulAssertionEvidence(currentState, testerConversation) ? 'finish' : 'stop'; + const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop'; return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator); } @@ -885,7 +885,12 @@ export class Pilot implements Agent { return parts.join('\n\n'); } - private hasSuccessfulAssertionEvidence(currentState: ActionResult, testerConversation: Conversation): boolean { + private hasCompletionEvidence(task: Test, currentState: ActionResult, testerConversation: Conversation): boolean { + if (task.hasAchievedAny()) return true; + return this.hasSuccessfulCheckEvidence(currentState, testerConversation); + } + + private hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean { if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true; return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful); } diff --git a/src/ai/tester.ts b/src/ai/tester.ts index b3e459a..cb703b7 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -136,31 +136,6 @@ export class Tester extends TaskAgent implements Agent { task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED); }); - const page = this.explorer.playwrightHelper?.page; - const observedPages = new Set(); - const onPageError = (err: Error) => { - task.addNote(`Console error: ${err.message}`, TestResult.FAILED); - }; - const onConsoleMessage = (msg: any) => { - if (msg.type() !== 'error') return; - task.addNote(`Console error: ${msg.text()}`, TestResult.FAILED); - }; - const watchPage = (targetPage: any) => { - if (!targetPage) return; - if (observedPages.has(targetPage)) return; - targetPage.on('pageerror', onPageError); - targetPage.on('console', onConsoleMessage); - observedPages.add(targetPage); - }; - const unwatchPages = () => { - for (const observedPage of observedPages) { - observedPage.off('pageerror', onPageError); - observedPage.off('console', onConsoleMessage); - } - observedPages.clear(); - }; - watchPage(page); - const initialState = ActionResult.fromState(state); const conversation = this.provider.startConversation(this.getSystemMessage(), 'tester'); @@ -190,19 +165,18 @@ export class Tester extends TaskAgent implements Agent { expected: task.expected, }, }, - async () => this.runTestSession(task, initialState, conversation, { offFailedRequest, watchPage, unwatchPages }) + async () => this.runTestSession(task, initialState, conversation, { offFailedRequest }) ); } private async runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers): Promise<{ success: boolean }> { - const { offFailedRequest, watchPage, unwatchPages } = handlers; + const { offFailedRequest } = handlers; if (this.pilot) { try { const plan = await this.pilot.planTest(task, initialState); if (task.hasFinished) { offFailedRequest?.(); - unwatchPages(); return { success: task.isSuccessful }; } if (plan) { @@ -214,43 +188,22 @@ export class Tester extends TaskAgent implements Agent { task.addNote(`Planning failed: ${message}`, TestResult.FAILED); task.finish(TestResult.FAILED); offFailedRequest?.(); - unwatchPages(); return { success: false }; } } debugLog('Starting test execution with tools'); - task.start(); - await this.explorer.startTest(task); - let initialSetupStopped = false; - if ( - !(await this.ensureBrowserPageAvailable( - task, - conversation, - () => { - initialSetupStopped = true; - }, - watchPage - )) - ) { - offFailedRequest?.(); - unwatchPages(); - await this.cleanupStartedTest(task); - return { success: task.isSuccessful }; - } - if (initialSetupStopped || task.hasFinished) { + if (!(await this.startExplorerTest(task, conversation))) { offFailedRequest?.(); - unwatchPages(); await this.cleanupStartedTest(task); return { success: task.isSuccessful }; } debugLog(`Navigating to ${task.startUrl}`); - const navigated = await this.visitStartUrlWithRecovery(task, conversation, watchPage); + const navigated = await this.visitStartUrlWithRecovery(task, conversation); if (!navigated) { offFailedRequest?.(); - unwatchPages(); await this.cleanupStartedTest(task); return { success: task.isSuccessful }; } @@ -278,7 +231,7 @@ export class Tester extends TaskAgent implements Agent { await loop( async ({ stop, pause, iteration, userInput }) => { debugLog('iteration', iteration); - if (!(await this.ensureBrowserPageAvailable(task, conversation, stop, watchPage))) return; + if (!(await this.ensureBrowserPageAvailable(task, conversation, stop))) return; const currentState = this.getCurrentState(); const tools = { @@ -426,27 +379,14 @@ export class Tester extends TaskAgent implements Agent { } : undefined, catch: async ({ error, stop }) => { - await this.handleExecutionError(task, conversation, error, stop, watchPage); + await this.handleExecutionError(task, conversation, error, stop); }, } ); if (task.hasFinished) break; - let finalReviewStopped = false; - if ( - !(await this.ensureBrowserPageAvailable( - task, - conversation, - () => { - finalReviewStopped = true; - }, - watchPage - )) - ) { - break; - } - if (finalReviewStopped || task.hasFinished) break; + if (!(await this.canRunFinalReview(task, conversation))) break; const finalState = this.getCurrentState(); const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator); @@ -476,7 +416,6 @@ export class Tester extends TaskAgent implements Agent { offStateChange(); offFailedRequest?.(); - unwatchPages(); await this.finishTest(task); await this.explorer.stopTest(task, { startUrl: task.startUrl, @@ -1138,7 +1077,7 @@ export class Tester extends TaskAgent implements Agent { }; } - private async handleExecutionError(task: Test, conversation: Conversation, error: Error, stop: () => void, watchPage: (page: any) => void): Promise { + private async handleExecutionError(task: Test, conversation: Conversation, error: Error, stop: () => void): Promise { tag('error').log(`Test execution error: ${error}`); const message = error instanceof Error ? error.message : String(error); if (!task.hasFinished) { @@ -1158,7 +1097,7 @@ export class Tester extends TaskAgent implements Agent { return; } if (recovery.recovered) { - watchPage(this.explorer.playwrightHelper?.page); + this.explorer.watchActiveTestPage(); this.resetFailureCount(); const recoveryContext = await this.buildRecoveryContext(task); conversation.addUserText(`${recovery.message}\n\n${recoveryContext}`); @@ -1185,6 +1124,18 @@ export class Tester extends TaskAgent implements Agent { }); } + private async startExplorerTest(task: Test, conversation: Conversation): Promise { + task.start(); + if (await this.explorer.startTest(task)) return true; + + let stopped = false; + await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), () => { + stopped = true; + }); + if (stopped) return false; + return !task.hasFinished; + } + private async buildRecoveryContext(task: Test): Promise { let currentState: ActionResult | null = null; try { @@ -1215,29 +1166,32 @@ export class Tester extends TaskAgent implements Agent { `; } - private async ensureBrowserPageAvailable(task: Test, conversation: Conversation, stop: () => void, watchPage: (page: any) => void): Promise { - const page = this.explorer.playwrightHelper?.page; - if (page && !page.isClosed?.()) return true; + private async ensureBrowserPageAvailable(task: Test, conversation: Conversation, stop: () => void): Promise { + if (await this.explorer.ensureActiveTestPageAvailable()) return true; - await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), stop, watchPage); + await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), stop); return !task.hasFinished; } - private async visitStartUrlWithRecovery(task: Test, conversation: Conversation, watchPage: (page: any) => void): Promise { + private async canRunFinalReview(task: Test, conversation: Conversation): Promise { + let stopped = false; + const available = await this.ensureBrowserPageAvailable(task, conversation, () => { + stopped = true; + }); + if (!available) return false; + if (stopped) return false; + return !task.hasFinished; + } + + private async visitStartUrlWithRecovery(task: Test, conversation: Conversation): Promise { try { await this.explorer.visit(task.startUrl!); return true; } catch (error) { let stopped = false; - await this.handleExecutionError( - task, - conversation, - error instanceof Error ? error : new Error(String(error)), - () => { - stopped = true; - }, - watchPage - ); + await this.handleExecutionError(task, conversation, error instanceof Error ? error : new Error(String(error)), () => { + stopped = true; + }); if (stopped || task.hasFinished) return false; @@ -1256,6 +1210,4 @@ export class Tester extends TaskAgent implements Agent { interface TestSessionHandlers { offFailedRequest?: () => void; - watchPage: (page: any) => void; - unwatchPages: () => void; } diff --git a/src/explorer.ts b/src/explorer.ts index 73ead20..1d54e84 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -5,10 +5,10 @@ import * as codeceptjs from 'codeceptjs'; import stepsListener from 'codeceptjs/lib/listener/steps'; import storeListener from 'codeceptjs/lib/listener/store'; import { createTest } from 'codeceptjs/lib/mocha/test'; +import type { BrowserContextOptions } from 'playwright'; import { ActionResult } from './action-result.ts'; import Action from './action.js'; import { AIProvider } from './ai/provider.js'; -import type { BrowserContextOptions } from 'playwright'; import { visuallyAnnotateContainers } from './ai/researcher/coordinates.ts'; import { RequestStore } from './api/request-store.ts'; import { XhrCapture } from './api/xhr-capture.ts'; @@ -19,7 +19,7 @@ import { KnowledgeTracker } from './knowledge-tracker.js'; import { PlaywrightRecorder } from './playwright-recorder.ts'; import { Reporter } from './reporter.ts'; import { StateManager } from './state-manager.js'; -import { Test } from './test-plan.ts'; +import { Test, TestResult } from './test-plan.ts'; import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts'; import { createDebug, log, tag } from './utils/logger.js'; import { WebElement } from './utils/web-element.ts'; @@ -64,6 +64,10 @@ class Explorer { private xhrCapture: XhrCapture | null = null; private requestStore: RequestStore | null = null; private playwrightRecorder: PlaywrightRecorder = new PlaywrightRecorder(); + private observedTestPages = new Set(); + private testPageErrorHandler: ((error: Error) => void) | null = null; + private testConsoleHandler: ((message: any) => void) | null = null; + private testDialogHandler: ((dialog: any) => void) | null = null; constructor(config: ExplorbotConfig, aiProvider: AIProvider, options?: { show?: boolean; headless?: boolean; incognito?: boolean; session?: string }) { this.config = config; @@ -617,11 +621,12 @@ class Explorer { return this._activeTest; } - async startTest(test: Test) { + async startTest(test: Test): Promise { this._activeTest = test; await this.reporter.reportTestStart(test); await this.closeOtherTabs(); this.otherTabs = []; + if (!(await this.ensureActiveTestPageAvailable())) return false; const codeceptjsTest = toCodeceptjsTest(test); @@ -638,13 +643,7 @@ class Explorer { test.setActiveNoteScreenshot(lastScreenshot); }; - const dialogHandler = (dialog: any) => { - const dialogType = dialog.type(); - const dialogMessage = dialog.message(); - test.addNote(`Native dialog ${dialogType} appeared: ${dialogMessage}. Accepted automatically`); - }; - - this.playwrightHelper?.page?.on('dialog', dialogHandler); + this.watchActiveTestPage(); codeceptjs.event.dispatcher.emit('test.before', codeceptjsTest); codeceptjs.event.dispatcher.emit('test.start', codeceptjsTest); @@ -655,11 +654,51 @@ class Explorer { codeceptjs.event.dispatcher.on('test.after', () => { codeceptjs.event.dispatcher.off('step.passed', stepHandler); codeceptjs.event.dispatcher.off('step.failed', stepHandler); - this.playwrightHelper?.page?.off('dialog', dialogHandler); + this.unwatchActiveTestPages(); }); + + return true; + } + + async ensureActiveTestPageAvailable(): Promise { + const page = this.playwrightHelper?.page; + if (page && !page.isClosed?.()) { + this.watchActiveTestPage(page); + return true; + } + + const recovered = await this.recoverFromBrowserError(); + if (!recovered) return false; + this.watchActiveTestPage(); + return true; + } + + watchActiveTestPage(page = this.playwrightHelper?.page): void { + if (!this._activeTest) return; + if (!page) return; + if (this.observedTestPages.has(page)) return; + + this.testPageErrorHandler ||= (err: Error) => { + this._activeTest?.addNote(`Console error: ${err.message}`, TestResult.FAILED); + }; + this.testConsoleHandler ||= (msg: any) => { + if (msg.type() !== 'error') return; + this._activeTest?.addNote(`Console error: ${msg.text()}`, TestResult.FAILED); + }; + this.testDialogHandler ||= (dialog: any) => { + const dialogType = dialog.type(); + const dialogMessage = dialog.message(); + this._activeTest?.addNote(`Native dialog ${dialogType} appeared: ${dialogMessage}. Accepted automatically`); + }; + + page.on('pageerror', this.testPageErrorHandler); + page.on('console', this.testConsoleHandler); + page.on('dialog', this.testDialogHandler); + this.observedTestPages.add(page); } async stopTest(test: Test, meta?: Record) { + this.unwatchActiveTestPages(); this._activeTest = null; const lastScreenshot = this.stateManager.getCurrentState()?.screenshotFile; if (lastScreenshot) { @@ -684,6 +723,15 @@ class Explorer { codeceptjs.event.dispatcher.emit('test.after', codeceptjsTest); } + private unwatchActiveTestPages(): void { + for (const page of this.observedTestPages) { + if (this.testPageErrorHandler) page.off('pageerror', this.testPageErrorHandler); + if (this.testConsoleHandler) page.off('console', this.testConsoleHandler); + if (this.testDialogHandler) page.off('dialog', this.testDialogHandler); + } + this.observedTestPages.clear(); + } + async hasPlaywrightLocator(locatorFn: (page: any) => any, opts: { multiple?: boolean; contents?: boolean; success?: (locator: any) => Promise | void } = {}): Promise { try { const pwLocator = locatorFn(this.playwrightHelper.page); diff --git a/tests/unit/captain-artifacts.test.ts b/tests/unit/captain-artifacts.test.ts index d7eaa21..a529354 100644 --- a/tests/unit/captain-artifacts.test.ts +++ b/tests/unit/captain-artifacts.test.ts @@ -1,7 +1,8 @@ +import { describe, expect, it } from 'bun:test'; import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { basename, dirname, join } from 'node:path'; -import { describe, expect, it } from 'bun:test'; import { Captain } from '../../src/ai/captain.ts'; +import { readCaptainFile } from '../../src/ai/captain/file-tools.ts'; import { ConfigParser } from '../../src/config.ts'; import type { Task } from '../../src/test-plan.ts'; @@ -52,7 +53,7 @@ describe('Captain artifact analysis tools', () => { const captain = buildCaptain(); const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); - const result = await tools.readArtifact.execute({ path: 'output/reports/session-demo-tests.md' }); + const result = await tools.readFile.execute({ path: 'output/reports/session-demo-tests.md' }); expect(result.success).toBe(true); expect(result.content).toContain('Expected button was missing'); @@ -72,13 +73,70 @@ describe('Captain artifact analysis tools', () => { const captain = buildCaptain(); const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); const projectName = basename(dirname(parser.getConfigPath()!)); - const result = await tools.readArtifact.execute({ path: `${projectName}/output/reports/session-demo-tests.md` }); + const result = await tools.readFile.execute({ path: `${projectName}/output/reports/session-demo-tests.md` }); expect(result.success).toBe(true); expect(result.content).toContain('Wrong expectation'); rmSync(join(outputDir, '..'), { recursive: true, force: true }); }); + + it('reads a requested line range from file contents', async () => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + const parser = ConfigParser.getInstance(); + const outputDir = join(dirname(parser.getConfigPath()!), 'output'); + const reportDir = join(outputDir, 'reports'); + mkdirSync(reportDir, { recursive: true }); + writeFileSync(join(reportDir, 'session-demo-tests.md'), ['line 1', 'line 2', 'line 3', 'line 4'].join('\n')); + + const captain = buildCaptain(); + const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); + const result = await tools.readFile.execute({ path: 'output/reports/session-demo-tests.md', startLine: 2, endLine: 3 }); + + expect(result.success).toBe(true); + expect(result.content).toBe('line 2\nline 3'); + + rmSync(join(outputDir, '..'), { recursive: true, force: true }); + }); + + it('reads line ranges from the end of file', async () => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + const parser = ConfigParser.getInstance(); + const outputDir = join(dirname(parser.getConfigPath()!), 'output'); + const reportDir = join(outputDir, 'reports'); + mkdirSync(reportDir, { recursive: true }); + writeFileSync(join(reportDir, 'session-demo-tests.md'), ['line 1', 'line 2', 'line 3', 'line 4'].join('\n')); + + const captain = buildCaptain(); + const tools = await (captain as any).idleModeTools({ explorBot: {}, task: task('analyze report') }); + const result = await tools.readFile.execute({ path: 'output/reports/session-demo-tests.md', startLine: -2 }); + + expect(result.success).toBe(true); + expect(result.content).toBe('line 3\nline 4'); + + rmSync(join(outputDir, '..'), { recursive: true, force: true }); + }); + + it('uses caller-provided readable directories', () => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + const parser = ConfigParser.getInstance(); + const projectRoot = dirname(parser.getConfigPath()!); + const customDir = join(projectRoot, 'custom-knowledge'); + mkdirSync(customDir, { recursive: true }); + writeFileSync(join(customDir, 'hint.md'), 'custom directory content'); + + const result = readCaptainFile(projectRoot, { path: 'custom-knowledge/hint.md' }, ['custom-knowledge']); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.content).toContain('custom directory content'); + } + + rmSync(projectRoot, { recursive: true, force: true }); + }); }); describe('Captain command guard', () => { diff --git a/tests/unit/pilot-evidence.test.ts b/tests/unit/pilot-evidence.test.ts index f6bf60f..82aa10a 100644 --- a/tests/unit/pilot-evidence.test.ts +++ b/tests/unit/pilot-evidence.test.ts @@ -11,7 +11,7 @@ describe('Pilot evidence', () => { const state = { verifications: { 'Heading is visible': true } }; const conversation = { getToolExecutions: () => [] }; - expect((pilot as any).hasSuccessfulAssertionEvidence(state, conversation)).toBe(true); + expect((pilot as any).hasSuccessfulCheckEvidence(state, conversation)).toBe(true); expect((pilot as any).formatSuccessfulAssertions(state, conversation)).toContain('PASS state verification'); }); @@ -29,7 +29,16 @@ describe('Pilot evidence', () => { ], }; - expect((pilot as any).hasSuccessfulAssertionEvidence(state, conversation)).toBe(true); + expect((pilot as any).hasSuccessfulCheckEvidence(state, conversation)).toBe(true); expect((pilot as any).formatSuccessfulAssertions(state, conversation)).toContain('PASS verify'); }); + + it('treats achieved task notes as completion evidence', () => { + const pilot = buildPilot(); + const task = { hasAchievedAny: () => true }; + const state = {}; + const conversation = { getToolExecutions: () => [] }; + + expect((pilot as any).hasCompletionEvidence(task, state, conversation)).toBe(true); + }); }); diff --git a/tests/unit/tester-execution-recovery.test.ts b/tests/unit/tester-execution-recovery.test.ts index dde11a5..9a324e4 100644 --- a/tests/unit/tester-execution-recovery.test.ts +++ b/tests/unit/tester-execution-recovery.test.ts @@ -8,6 +8,8 @@ function buildTester(captain?: any, page: any = { id: 'recovered-page' }, explor playwrightHelper: { page, }, + ensureActiveTestPageAvailable: async () => !!page && !page.isClosed?.(), + watchActiveTestPage: () => {}, createAction: () => ({ capturePageState: async () => ({ url: '/', @@ -78,17 +80,12 @@ describe('Tester execution recovery', () => { const task = buildTask(); const conversation = buildConversation(); const watchedPages: any[] = []; + (tester as any).explorer.watchActiveTestPage = () => watchedPages.push((tester as any).explorer.playwrightHelper.page); let stopped = false; - await (tester as any).handleExecutionError( - task, - conversation, - new Error('Target closed'), - () => { - stopped = true; - }, - (page: any) => watchedPages.push(page) - ); + await (tester as any).handleExecutionError(task, conversation, new Error('Target closed'), () => { + stopped = true; + }); expect(stopped).toBe(false); expect(task.hasFinished).toBe(false); @@ -110,15 +107,9 @@ describe('Tester execution recovery', () => { const conversation = buildConversation(); let stopped = false; - await (tester as any).handleExecutionError( - task, - conversation, - new Error('Target closed'), - () => { - stopped = true; - }, - () => {} - ); + await (tester as any).handleExecutionError(task, conversation, new Error('Target closed'), () => { + stopped = true; + }); expect(stopped).toBe(true); expect(task.hasFinished).toBe(true); @@ -132,15 +123,9 @@ describe('Tester execution recovery', () => { const conversation = buildConversation(); let stopped = false; - await (tester as any).handleExecutionError( - task, - conversation, - new Error('Locator not found'), - () => { - stopped = true; - }, - () => {} - ); + await (tester as any).handleExecutionError(task, conversation, new Error('Locator not found'), () => { + stopped = true; + }); expect(stopped).toBe(false); expect(conversation.messages[0]).toContain('Previous AI call failed'); @@ -163,16 +148,12 @@ describe('Tester execution recovery', () => { const task = buildTask(); const conversation = buildConversation(); const watchedPages: any[] = []; + (tester as any).explorer.watchActiveTestPage = () => watchedPages.push((tester as any).explorer.playwrightHelper.page); let stopped = false; - const available = await (tester as any).ensureBrowserPageAvailable( - task, - conversation, - () => { - stopped = true; - }, - (page: any) => watchedPages.push(page) - ); + const available = await (tester as any).ensureBrowserPageAvailable(task, conversation, () => { + stopped = true; + }); expect(available).toBe(true); expect(stopped).toBe(false); @@ -205,14 +186,12 @@ describe('Tester execution recovery', () => { const task = buildTask(); task.startUrl = '/'; const conversation = buildConversation(); - const watchedPages: any[] = []; - const navigated = await (tester as any).visitStartUrlWithRecovery(task, conversation, (page: any) => watchedPages.push(page)); + const navigated = await (tester as any).visitStartUrlWithRecovery(task, conversation); expect(navigated).toBe(true); expect(visits).toBe(2); expect(task.hasFinished).toBe(false); - expect(watchedPages).toHaveLength(1); }); it('cleans up started test lifecycle on early startup failure', async () => { From fce5c15fdeb4e5b9925b74d6bf93baaafef0b445 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Tue, 9 Jun 2026 16:05:49 +0300 Subject: [PATCH 5/6] update captain and recovery logic --- src/action.ts | 45 +++- src/ai/captain.ts | 83 +++---- src/ai/navigator.ts | 8 +- src/ai/pilot.ts | 6 +- src/ai/researcher.ts | 29 ++- src/ai/researcher/coordinates.ts | 5 +- src/ai/researcher/deep-analysis.ts | 12 +- src/ai/researcher/locators.ts | 3 +- src/ai/task-agent.ts | 2 +- src/ai/tester.ts | 162 +++---------- src/ai/tools.ts | 24 +- src/explorer.ts | 161 +++++++++++-- src/utils/browser-errors.ts | 25 ++ tests/integration/researcher-browser.test.ts | 3 + tests/integration/researcher-sections.test.ts | 3 + tests/integration/researcher.test.ts | 3 + tests/unit/captain-artifacts.test.ts | 36 ++- tests/unit/captain-mode.test.ts | 31 ++- tests/unit/explorer-recovery-url.test.ts | 95 ++++++++ tests/unit/tester-execution-recovery.test.ts | 215 ------------------ 20 files changed, 484 insertions(+), 467 deletions(-) create mode 100644 src/utils/browser-errors.ts delete mode 100644 tests/unit/tester-execution-recovery.test.ts diff --git a/src/action.ts b/src/action.ts index 0d1bbe2..31a32f3 100644 --- a/src/action.ts +++ b/src/action.ts @@ -23,9 +23,9 @@ import { htmlCombinedSnapshot, minifyHtml } from './utils/html.js'; import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; import { safeFilename } from './utils/strings.ts'; import { throttle } from './utils/throttle.ts'; +import { isFatalBrowserError } from './utils/browser-errors.ts'; const debugLog = createDebug('explorbot:action'); -const FATAL_BROWSER_ERRORS = /Frame was detached|Target closed|Execution context was destroyed|Protocol error|Session closed/i; class Action { private actor: CodeceptJS.I; @@ -78,21 +78,26 @@ class Action { const page = this.playwrightHelper.page; const frame = this.playwrightHelper.frame; await page?.waitForLoadState('domcontentloaded', { timeout: 10000 })?.catch(() => {}); - const grabAll = () => Promise.all([(this.actor as any).grabSource(), (this.actor as any).grabTitle(), this.captureBrowserLogs()]); + await waitForUsablePageDom(page); + const grabAll = () => Promise.all([captureHtml(page, frame), captureTitle(page), this.captureBrowserLogs()]); const [html, title, browserLogs] = await grabAll().catch(async (err: Error) => { const msg = err instanceof Error ? err.message : String(err); if (!/navigating and changing the content/i.test(msg)) throw err; await page?.waitForLoadState('domcontentloaded', { timeout: 10000 })?.catch(() => {}); + await waitForUsablePageDom(page); return grabAll(); }); const url = page?.url() || (await (this.actor as any).grabCurrentUrl?.()); let screenshotFile: string | undefined = undefined; + const statesDir = outputPath('states'); + fs.mkdirSync(statesDir, { recursive: true }); if (includeScreenshot) { const filename = safeFilename(`${stateHash}_${timestamp}`, '.png'); - screenshotFile = await (this.actor as any) - .saveScreenshot(filename) + const screenshotPath = join(statesDir, filename); + screenshotFile = await page + ?.screenshot({ path: screenshotPath, fullPage: true }) .then(() => filename) .catch((err: Error) => { debugLog('Screenshot failed, continuing without it:', err); @@ -101,8 +106,6 @@ class Action { } // Save HTML to file - const statesDir = outputPath('states'); - fs.mkdirSync(statesDir, { recursive: true }); const htmlFile = safeFilename(`${stateHash}_${timestamp}`, '.html'); const htmlPath = join(statesDir, htmlFile); fs.writeFileSync(htmlPath, html, 'utf8'); @@ -158,7 +161,7 @@ class Action { return result; } catch (err) { const msg = err instanceof Error ? err.message : String(err); - if (FATAL_BROWSER_ERRORS.test(msg)) throw err; + if (isFatalBrowserError(err)) throw err; debugLog('capturePageState failed with non-fatal error:', msg); const url = this.playwrightHelper.page?.url?.() || ''; return new ActionResult({ url, error: msg }); @@ -375,6 +378,7 @@ class Action { return true; } catch (error) { this.lastError = error as Error; + if (isFatalBrowserError(error)) throw error; debugLog(`Attempt failed: ${codeBlock}: ${errorToString(error) || this.lastError?.toString()}`); return false; } @@ -406,6 +410,33 @@ function errorToString(error: any): string { return error.message || error.toString(); } +async function waitForUsablePageDom(page: any): Promise { + if (!page?.waitForFunction) return; + + await page + .waitForFunction( + () => { + const body = document.body; + if (!body) return false; + return body.children.length > 0 || body.textContent?.trim().length > 0; + }, + undefined, + { timeout: 5000 } + ) + .catch(() => {}); +} + +async function captureHtml(page: any, frame: any): Promise { + if (frame?.content) return frame.content(); + if (page?.content) return page.content(); + throw new Error('Playwright page is unavailable for HTML capture'); +} + +async function captureTitle(page: any): Promise { + if (page?.title) return page.title(); + return ''; +} + function sanitizeCodeBlock(code: string): string { return code .split('\n') diff --git a/src/ai/captain.ts b/src/ai/captain.ts index bb7be65..bb5756b 100644 --- a/src/ai/captain.ts +++ b/src/ai/captain.ts @@ -26,7 +26,6 @@ const MAX_STEPS = 15; const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent as unknown as new (...args: any[]) => TaskAgent))); export class Captain extends CaptainBase implements Agent { - protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate']; emoji = '🧑‍✈️'; private explorBot: ExplorBot; private conversation: Conversation | null = null; @@ -72,6 +71,12 @@ export class Captain extends CaptainBase implements Agent { protected trackToolExecutions(toolExecutions: any[]): void { super.trackToolExecutions(toolExecutions); + if (toolExecutions.length > 0) { + this.recentToolCalls.push(...toolExecutions); + if (this.recentToolCalls.length > 20) { + this.recentToolCalls = this.recentToolCalls.slice(-20); + } + } for (const exec of toolExecutions) { const label = toolExecutionLabel(exec.input); if (!label) continue; @@ -117,7 +122,7 @@ export class Captain extends CaptainBase implements Agent { - After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification - Prefer completing in fewer tool calls over thoroughness - NEVER run tests unless the user explicitly asks - - If the user asks to show, display, explain, summarize, compare, or diagnose information, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained. + - If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained. ${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''} ${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''} ${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''} @@ -262,10 +267,10 @@ export class Captain extends CaptainBase implements Agent { }), execute: async ({ summary, details }) => { debugLog('done', summary); - if (requiresUserFacingDetails(task.description) && !details?.trim()) { + if (!details?.trim() && !this.canCompleteWithoutDetails()) { return { success: false, - message: 'The user asked for information to be shown or explained. Call done() again with the actual answer in details, not only a completion summary.', + message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.', }; } if (details?.trim()) { @@ -282,7 +287,7 @@ export class Captain extends CaptainBase implements Agent { Execute a TUI command. Returns log output from command execution. Use only when the user explicitly asks to run a slash command. Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience. - Never run /test, /rerun, /plan, /plans, /plan:load, /experience, /learn, or /drill unless the user explicitly requested that slash command. + Never run a slash command unless the user request itself starts with that slash command. ${this.commandDescriptions .map((c) => { const opts = c.options ? ` (${c.options})` : ''; @@ -296,11 +301,11 @@ export class Captain extends CaptainBase implements Agent { execute: async ({ command }) => { if (!this.commandExecutor) return { success: false, message: 'Command executor not available' }; const cmd = command.startsWith('/') ? command : `/${command}`; - if (isUnsafeImplicitCommand(cmd) && !isExplicitSlashRequest(task.description, cmd)) { + if (!isExplicitSlashRequest(task.description, cmd)) { return { success: false, command: cmd, - message: 'Command blocked: slash commands that can run tests, mutate plans, or inspect unrelated project sections require an explicit user slash-command request. Use readFile/project/test inspection tools for analysis instead.', + message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.', }; } startLogCapture(); @@ -396,40 +401,18 @@ export class Captain extends CaptainBase implements Agent { } async processExecutionError(error: Error, activeTest: Test): Promise { - const message = error.message || String(error); const explorer = this.explorBot.getExplorer(); - - if (!explorer.isFatalBrowserError(error)) { - return { - action: 'continue', - message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, - }; - } - - let recovered = await explorer.recoverFromBrowserError(); - if (!recovered) { - recovered = await explorer.restartBrowser(); - } - if (recovered) { - return { - action: 'continue', - recovered: true, - message: dedent` - Captain recovered the browser after a fatal page error. - Continue the test "${activeTest.scenario}" from the restored page. - The interrupted action is not evidence that the application failed. - Inspect the restored page and retry the scenario step when it is still required. - `, - }; - } - + const result = await explorer.handleExecutionError(error); return { - action: 'stop', - recovered: false, - message: `Captain could not recover the browser after fatal error: ${message}`, + ...result, + message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message, }; } + private canCompleteWithoutDetails(): boolean { + return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence); + } + async handle(input: string, options: { reset?: boolean } = {}): Promise { const stateManager = this.explorBot.getExplorer().getStateManager(); const initialState = stateManager.getCurrentState(); @@ -525,7 +508,7 @@ export class Captain extends CaptainBase implements Agent { if (result?.toolExecutions?.length) { const lastExec = result.toolExecutions[result.toolExecutions.length - 1]; - if (lastExec.wasSuccessful && this.ACTION_TOOLS.includes(lastExec.toolName)) { + if (hasBrowserCompletionEvidence(lastExec)) { conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.'); } } @@ -569,19 +552,25 @@ interface ExecutionRecoveryAction { recovered?: boolean; } -function isUnsafeImplicitCommand(command: string): boolean { - return /^\/(?:test|rerun|plan(?::load)?|plans|experience|learn|drill)(?:\s|$)/.test(command.trim()); +function isExplicitSlashRequest(input: string, command: string): boolean { + const requested = slashCommandToken(input); + const actual = slashCommandToken(command); + if (!requested || !actual) return false; + return requested === actual; } -function isExplicitSlashRequest(input: string, command: string): boolean { - const normalizedInput = input.trim(); - if (!normalizedInput.startsWith('/')) return false; +function slashCommandToken(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed.startsWith('/')) return null; - const requested = normalizedInput.split(/\s+/)[0]; - const actual = command.trim().split(/\s+/)[0]; - return requested === actual; + for (let i = 1; i < trimmed.length; i++) { + if (trimmed[i] <= ' ') return trimmed.slice(0, i); + } + return trimmed; } -function requiresUserFacingDetails(input: string): boolean { - return /\b(show|display|explain|describe|summarize|analyse|analyze|compare|tell me|details|status|what|which|why|how|где|покажи|объясни|обьясни|расскажи|что|какой|какие|почему|как)\b/i.test(input); +function hasBrowserCompletionEvidence(execution: any): boolean { + if (!execution?.wasSuccessful) return false; + const output = execution.output || {}; + return Boolean(output.pageDiff || output.code || output.playwrightGroupId); } diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 215e056..9e027ee 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -136,6 +136,10 @@ class Navigator implements Agent { } async visit(url: string): Promise { + return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url)); + } + + private async visitOnce(url: string): Promise { try { const action = this.explorer.createAction(); @@ -170,7 +174,7 @@ class Navigator implements Agent { throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`); } } - await action.caputrePageWithScreenshot(); + await this.explorer.capturePageWithScreenshot(); await this.hooksRunner.runAfterHook('navigator', url); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -382,7 +386,7 @@ class Navigator implements Agent { // URL did not transition to expectedUrl within timeout } } - const freshState = await action.capturePageState(); + const freshState = await this.explorer.capturePageState(); const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || ''; const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl); const stateChanged = freshState.getStateHash() !== actionResult.getStateHash(); diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 128f1b0..72698bf 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -93,8 +93,7 @@ export class Pilot implements Agent { let screenshotState: ActionResult | null = null; if (this.provider.hasVision()) { try { - const action = this.explorer.createAction(); - screenshotState = await action.caputrePageWithScreenshot(); + screenshotState = await this.explorer.capturePageWithScreenshot(); if (screenshotState.screenshot) { visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || ''; } @@ -661,8 +660,7 @@ export class Pilot implements Agent { private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise { if (!this.provider.hasVision()) return null; - const action = this.explorer.createAction(); - const screenshotState = await action.caputrePageWithScreenshot().catch(() => null); + const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null); if (!screenshotState?.screenshot) return null; const question = dedent` diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 5d3c4cf..db39d35 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -130,7 +130,7 @@ export class Researcher extends ResearcherBase implements Agent { const annotatedElements = await this.explorer.annotateElements(); debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`); - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() }); const condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -239,7 +239,7 @@ export class Researcher extends ResearcherBase implements Agent { // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring. if (!interrupted() && this.hasScreenshotToAnalyze) { const sections = parseResearchSections(result.text); - const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections); + const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections)); if (focused) markSectionAsFocused(result, focused); } @@ -252,7 +252,7 @@ export class Researcher extends ResearcherBase implements Agent { const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator); const containers = validContainers.filter((c) => !freshBroken.includes(c.css)); await this.visuallyAnnotateElements({ containers }); - this.actionResult = await this.explorer.createAction().caputrePageWithScreenshot(); + this.actionResult = await this.explorer.capturePageWithScreenshot(); const visualResult = await this.analyzeScreenshotForVisualProps(); if (visualResult.elements.size > 0) { await this.mergeVisualData(result, visualResult.elements); @@ -331,7 +331,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!this.actionResult) { debugLog('No action result, navigating to URL'); await this.explorer.visit(url); - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); return; } @@ -341,7 +341,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!isEmpty && isOnCurrentState) { if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) { - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot }); } return; } @@ -349,6 +349,8 @@ export class Researcher extends ResearcherBase implements Agent { if (isEmpty && isOnCurrentState) { debugLog('HTML body empty on current URL, waiting for content'); tag('step').log('Page body is empty, waiting for content...'); + await this.explorer.visit(url); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); await this.waitUntilSettled(screenshot ?? false); return; } @@ -357,22 +359,21 @@ export class Researcher extends ResearcherBase implements Agent { tag('step').log('Navigating to URL...'); await this.explorer.visit(url); - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot ?? false }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false }); } private async waitUntilSettled(screenshot: boolean): Promise { const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10; if (errorPageTimeout <= 0) return false; - const page = this.explorer.playwrightHelper.page; const includeScreenshot = screenshot && this.provider.hasVision(); try { - await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }); + await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 })); } catch {} await this.explorer.annotateElements(); - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); let condition = detectPageCondition(this.actionResult!); if (condition === 'error') { @@ -383,7 +384,7 @@ export class Researcher extends ResearcherBase implements Agent { for (let i = 0; i < 3; i++) { await new Promise((r) => setTimeout(r, 1000)); await this.explorer.annotateElements(); - this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot }); + this.actionResult = await this.explorer.capturePageState({ includeScreenshot }); condition = detectPageCondition(this.actionResult!); if (condition === 'error') { throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title); @@ -762,17 +763,15 @@ export class Researcher extends ResearcherBase implements Agent { } async navigateTo(url: string): Promise { - const action = this.explorer.createAction(); - await action.execute(`I.amOnPage("${url}")`); + await this.explorer.visit(url); } async cancelInUi() { const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null; - const action = this.explorer.createAction(); - await action.execute('I.clickXY(0, 0)'); + await this.explorer.executeAction('I.clickXY(0, 0)'); if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null)) return; - await action.execute(`I.pressKey('Escape')`); + await this.explorer.executeAction(`I.pressKey('Escape')`); } } diff --git a/src/ai/researcher/coordinates.ts b/src/ai/researcher/coordinates.ts index 9a45f04..3893fca 100644 --- a/src/ai/researcher/coordinates.ts +++ b/src/ai/researcher/coordinates.ts @@ -81,7 +81,7 @@ export function WithCoordinates(Base: T) { } async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return visuallyAnnotateContainers(this.explorer.playwrightHelper.page, opts?.containers || []); + return this.explorer.visuallyAnnotateElements(opts); } private async _analyzeScreenshotForVisualProps(): Promise { @@ -193,7 +193,6 @@ export function WithCoordinates(Base: T) { } async backfillCoordinates(result: ResearchResult): Promise { - const page = this.explorer.playwrightHelper.page; const sections = parseResearchSections(result.text); const eidxWithoutCoords: string[] = []; for (const section of sections) { @@ -203,7 +202,7 @@ export function WithCoordinates(Base: T) { } if (eidxWithoutCoords.length === 0) return; - const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords); + const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords)); if (webElements.length === 0) return; const rectMap = new Map(webElements.map((w) => [w.eidx!, w])); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index 1f7296b..cebe84b 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -279,11 +279,10 @@ export function WithDeepAnalysis(Base: T) { const isCoordinateClick = el.commands[0].startsWith('I.clickXY('); if (!isCoordinateClick) { const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo('); - const hoverAction = this.explorer.createAction(); - await hoverAction.attempt(hoverCmd, undefined, false); + await this.explorer.attemptAction(hoverCmd, undefined, false); await new Promise((r) => setTimeout(r, 500)); - await this.explorer.createAction().capturePageState(); + await this.explorer.capturePageState(); const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!); const hoverDiff = await hoverAR.diff(previousState); await hoverDiff.calculate(); @@ -303,9 +302,8 @@ export function WithDeepAnalysis(Base: T) { } let clickCode: string | null = null; - const action = this.explorer.createAction(); for (const cmd of el.commands) { - if (await action.attempt(cmd, undefined, false)) { + if (await this.explorer.attemptAction(cmd, undefined, false)) { clickCode = cmd; break; } @@ -319,7 +317,7 @@ export function WithDeepAnalysis(Base: T) { let diff: Diff; try { - await this.explorer.createAction().capturePageState(); + await this.explorer.capturePageState(); const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!); diff = await currAR.diff(previousState); await diff.calculate(); @@ -361,7 +359,7 @@ export function WithDeepAnalysis(Base: T) { private async _restorePageState(url: string, originalAria: string): Promise { try { await (this as any).cancelInUi(); - await this.explorer.createAction().capturePageState(); + await this.explorer.capturePageState(); const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || ''; if (!diffAriaSnapshots(originalAria, currentAria)) return; } catch (err) { diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index fea3c46..0406693 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -194,8 +194,7 @@ export function WithLocators(Base: T) { } if (needsXpath.length > 0) { - const page = this.explorer.playwrightHelper.page; - const webElements = await WebElement.fromEidxList(page, needsXpath); + const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath)); const changedSections = new Set<(typeof sections)[0]>(); for (const w of webElements) { const entry = needsXpathEls.get(w.eidx!); diff --git a/src/ai/task-agent.ts b/src/ai/task-agent.ts index efe918a..08dc966 100644 --- a/src/ai/task-agent.ts +++ b/src/ai/task-agent.ts @@ -24,7 +24,7 @@ export abstract class TaskAgent { protected consecutiveFailures = 0; protected consecutiveEmptyResults = 0; protected recentToolCalls: any[] = []; - protected abstract readonly ACTION_TOOLS: string[]; + protected readonly ACTION_TOOLS: string[] = []; private _historian: Historian | null = null; private _quartermaster: Quartermaster | null = null; diff --git a/src/ai/tester.ts b/src/ai/tester.ts index cb703b7..f1cb30a 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -194,18 +194,22 @@ export class Tester extends TaskAgent implements Agent { debugLog('Starting test execution with tools'); - if (!(await this.startExplorerTest(task, conversation))) { + if (!(await this.explorer.startTest(task))) { offFailedRequest?.(); await this.cleanupStartedTest(task); return { success: task.isSuccessful }; } debugLog(`Navigating to ${task.startUrl}`); - const navigated = await this.visitStartUrlWithRecovery(task, conversation); - if (!navigated) { - offFailedRequest?.(); - await this.cleanupStartedTest(task); - return { success: task.isSuccessful }; + try { + await this.explorer.visit(task.startUrl!); + } catch (error) { + const result = await this.handleLoopError(task, error); + if (result === 'stop') { + offFailedRequest?.(); + await this.cleanupStartedTest(task); + return { success: task.isSuccessful }; + } } const startState = this.explorer.getStateManager().getCurrentState(); @@ -231,7 +235,12 @@ export class Tester extends TaskAgent implements Agent { await loop( async ({ stop, pause, iteration, userInput }) => { debugLog('iteration', iteration); - if (!(await this.ensureBrowserPageAvailable(task, conversation, stop))) return; + if (!(await this.explorer.ensurePageAvailable())) { + task.addNote('Browser page is unavailable'); + task.finish(TestResult.FAILED); + stop(); + return; + } const currentState = this.getCurrentState(); const tools = { @@ -379,14 +388,15 @@ export class Tester extends TaskAgent implements Agent { } : undefined, catch: async ({ error, stop }) => { - await this.handleExecutionError(task, conversation, error, stop); + const result = await this.handleLoopError(task, error); + if (result === 'stop') stop(); }, } ); if (task.hasFinished) break; - if (!(await this.canRunFinalReview(task, conversation))) break; + if (!(await this.explorer.ensurePageAvailable())) break; const finalState = this.getCurrentState(); const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator); @@ -1077,42 +1087,27 @@ export class Tester extends TaskAgent implements Agent { }; } - private async handleExecutionError(task: Test, conversation: Conversation, error: Error, stop: () => void): Promise { - tag('error').log(`Test execution error: ${error}`); + private async handleLoopError(task: Test, error: unknown): Promise<'continue' | 'stop'> { const message = error instanceof Error ? error.message : String(error); - if (!task.hasFinished) { - task.addNote(`Execution error: ${message}`); - } - if (error instanceof Error && error.name === 'AbortError') { - stop(); - return; - } - if (this.captain && error instanceof Error) { - const recovery = await this.captain.processExecutionError(error, task); - tag('info').log(`Supervisor: ${recovery.action} - ${recovery.message}`); - task.addNote(recovery.message); - if (recovery.action === 'stop') { - task.finish(TestResult.FAILED); - stop(); - return; - } - if (recovery.recovered) { - this.explorer.watchActiveTestPage(); - this.resetFailureCount(); - const recoveryContext = await this.buildRecoveryContext(task); - conversation.addUserText(`${recovery.message}\n\n${recoveryContext}`); - return; - } - conversation.addUserText(recovery.message); - return; - } - const isFatalBrowserError = this.explorer.isFatalBrowserError?.(error) ?? /Target closed|Session closed|Protocol error/i.test(message); - if (isFatalBrowserError) { + if (!task.hasFinished) task.addNote(`Execution error: ${message}`); + + const result = await this.explorer.handleExecutionError(error); + tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`); + task.addNote(result.message); + + if (result.action === 'stop') { task.finish(TestResult.FAILED); - stop(); - return; + return 'stop'; } - conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`); + + if (result.recovered) { + this.resetFailureCount(); + this.previousUrl = null; + this.previousStateHash = null; + } + + this.currentConversation?.addUserText(result.message); + return 'continue'; } private async cleanupStartedTest(task: Test): Promise { @@ -1123,89 +1118,6 @@ export class Tester extends TaskAgent implements Agent { sessionName: task.sessionName, }); } - - private async startExplorerTest(task: Test, conversation: Conversation): Promise { - task.start(); - if (await this.explorer.startTest(task)) return true; - - let stopped = false; - await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), () => { - stopped = true; - }); - if (stopped) return false; - return !task.hasFinished; - } - - private async buildRecoveryContext(task: Test): Promise { - let currentState: ActionResult | null = null; - try { - currentState = await this.explorer.createAction().capturePageState(); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return dedent` - - Browser was restored, but fresh page context could not be captured: ${message} - The previous Target closed error was an external browser interruption, not product evidence. - Re-open or inspect the page before deciding pass/fail. - - `; - } - - this.previousUrl = null; - this.previousStateHash = null; - const pageContext = await this.reinjectContextIfNeeded(1, currentState); - return dedent` - - Browser was restored during "${task.scenario}". - The previous Target closed error was an external browser interruption, not product evidence. - Ignore interrupted click/form attempts when judging whether the application works. - Retry the current scenario step from the restored page if the goal is still incomplete. - - - ${pageContext} - `; - } - - private async ensureBrowserPageAvailable(task: Test, conversation: Conversation, stop: () => void): Promise { - if (await this.explorer.ensureActiveTestPageAvailable()) return true; - - await this.handleExecutionError(task, conversation, new Error('Target closed: browser page is unavailable'), stop); - return !task.hasFinished; - } - - private async canRunFinalReview(task: Test, conversation: Conversation): Promise { - let stopped = false; - const available = await this.ensureBrowserPageAvailable(task, conversation, () => { - stopped = true; - }); - if (!available) return false; - if (stopped) return false; - return !task.hasFinished; - } - - private async visitStartUrlWithRecovery(task: Test, conversation: Conversation): Promise { - try { - await this.explorer.visit(task.startUrl!); - return true; - } catch (error) { - let stopped = false; - await this.handleExecutionError(task, conversation, error instanceof Error ? error : new Error(String(error)), () => { - stopped = true; - }); - - if (stopped || task.hasFinished) return false; - - try { - await this.explorer.visit(task.startUrl!); - return true; - } catch (retryError) { - const message = retryError instanceof Error ? retryError.message : String(retryError); - task.addNote(`Initial navigation failed after recovery: ${message}`, TestResult.FAILED); - task.finish(TestResult.FAILED); - return false; - } - } - } } interface TestSessionHandlers { diff --git a/src/ai/tools.ts b/src/ai/tools.ts index c7fe136..adc3b03 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -6,6 +6,7 @@ import type { ExperienceTracker } from '../experience-tracker.ts'; import type Explorer from '../explorer.ts'; import { type Task, TestResult } from '../test-plan.js'; import { extractFocusedElement } from '../utils/aria.ts'; +import { isFatalBrowserError } from '../utils/browser-errors.ts'; import { createDebug, tag } from '../utils/logger.js'; import { pause } from '../utils/loop.js'; import { WebElement } from '../utils/web-element.ts'; @@ -287,6 +288,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { suggestion: 'Verify the key name is correct. For typing text, use form() tool instead.', }); } catch (error) { + throwIfFatalBrowserError(error); activeNote.commit(TestResult.FAILED); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('pressKey', `PressKey tool failed: ${errorMessage}`); @@ -403,6 +405,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) { action ); } catch (error) { + throwIfFatalBrowserError(error); activeNote.commit(TestResult.FAILED); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('form', `Form tool failed: ${errorMessage}`); @@ -442,8 +445,7 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe') await explorer.switchToMainFrame(); - const action = explorer.createAction(); - const nextState = await action.capturePageState(); + const nextState = await explorer.capturePageState(); const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()'); return successToolResult('exitIframe', { @@ -452,6 +454,7 @@ export function createSpecialContextTools(explorer: Explorer, context: 'iframe') code: 'I.switchTo()', }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('exitIframe', `Failed to exit iframe: ${errorMessage}`); } @@ -496,8 +499,7 @@ export function createAgentTools({ } try { - const action = explorer.createAction(); - const actionResult = await action.caputrePageWithScreenshot(); + const actionResult = await explorer.capturePageWithScreenshot(); if (!actionResult.screenshot) { return failedToolResult('see', 'Failed to capture screenshot for analysis'); @@ -515,6 +517,7 @@ export function createAgentTools({ suggestion: 'Visual confirmation is valid evidence for test results. Use record() to note the visual findings.', }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; visionDisabled = true; tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.'); @@ -596,8 +599,7 @@ export function createAgentTools({ }); } - const action = explorer.createAction(); - const actionResult = await action.capturePageState(); + const actionResult = await explorer.capturePageState(); const result = await navigator.verifyState(assertion, actionResult); if (result.verified) { @@ -615,6 +617,7 @@ export function createAgentTools({ suggestion: 'The assertion could not be verified. Check if the condition is actually present on the page or try a different assertion.', }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('verify', `Verify tool failed: ${errorMessage}`, { error: errorMessage, @@ -670,6 +673,7 @@ export function createAgentTools({ `, }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('research', `Research tool failed: ${errorMessage}`, { error: errorMessage, @@ -714,6 +718,7 @@ export function createAgentTools({ suggestion: 'The action could not be completed. Try a different instruction or use more specific element descriptions.', }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; return failedToolResult('interact', `Interact tool failed: ${errorMessage}`, { error: errorMessage, @@ -752,7 +757,7 @@ export function createAgentTools({ const previousState = ActionResult.fromState(currentState); const action = explorer.createAction(); - const actionResult = await action.caputrePageWithScreenshot(); + const actionResult = await explorer.capturePageWithScreenshot(); if (!actionResult.screenshot) { return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis'); @@ -793,6 +798,7 @@ export function createAgentTools({ analysis: locationResult, }); } catch (error) { + throwIfFatalBrowserError(error); const errorMessage = error instanceof Error ? error.toString() : 'Unknown error occurred'; visionDisabled = true; tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.'); @@ -1012,6 +1018,10 @@ function cap(text: string | undefined | null, max: number): string { return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`; } +function throwIfFatalBrowserError(error: unknown): void { + if (isFatalBrowserError(error)) throw error; +} + function transformContainsCommand(command: string): string { if (!command.includes(':contains(')) return command; diff --git a/src/explorer.ts b/src/explorer.ts index 1d54e84..7838381 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync } from 'node:fs'; import path, { join } from 'node:path'; // @ts-ignore import * as codeceptjs from 'codeceptjs'; +import dedent from 'dedent'; import stepsListener from 'codeceptjs/lib/listener/steps'; import storeListener from 'codeceptjs/lib/listener/store'; import { createTest } from 'codeceptjs/lib/mocha/test'; @@ -23,6 +24,7 @@ import { Test, TestResult } from './test-plan.ts'; import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts'; import { createDebug, log, tag } from './utils/logger.js'; import { WebElement } from './utils/web-element.ts'; +import { BrowserRecoveryError, isFatalBrowserError } from './utils/browser-errors.ts'; declare global { namespace NodeJS { @@ -39,7 +41,6 @@ declare namespace CodeceptJS { } const debugLog = createDebug('explorbot:explorer'); -const FATAL_BROWSER_ERRORS = /Frame was detached|Target closed|Execution context was destroyed|Protocol error|Session closed/i; const RECOVERABLE_NAVIGATION_ERRORS = /net::ERR_ABORTED|page\.screenshot.*Timeout|waiting for fonts to load/i; interface TabInfo { @@ -47,6 +48,12 @@ interface TabInfo { title: string; } +interface BrowserExecutionErrorResult { + action: 'continue' | 'stop'; + message: string; + recovered?: boolean; +} + class Explorer { private aiProvider: AIProvider; playwrightHelper: any; @@ -299,11 +306,58 @@ class Explorer { return new Action(this.actor, this.stateManager, this.playwrightRecorder); } + async runWithBrowserRecovery(label: string, operation: () => Promise): Promise { + if (!(await this.ensurePageAvailable())) { + throw new Error(`Browser page is unavailable before ${label}`); + } + + try { + return await operation(); + } catch (error) { + if (!this.isFatalBrowserError(error)) throw error; + + tag('warning').log(`${label}: browser page is unavailable, recovering...`); + let recovered = await this.recoverFromBrowserError(); + if (!recovered) recovered = await this.restartBrowser(); + if (!recovered) throw new BrowserRecoveryError(label, error, false); + if (!(await this.waitForUsablePageDom())) throw new BrowserRecoveryError(label, error, true); + + try { + return await operation(); + } catch (retryError) { + if (this.isFatalBrowserError(retryError)) { + throw new BrowserRecoveryError(label, retryError, true); + } + throw retryError; + } + } + } + + async capturePageState(opts: { includeScreenshot?: boolean } = {}): Promise { + return this.runWithBrowserRecovery('capturePageState', () => this.createAction().capturePageState(opts)); + } + + async capturePageWithScreenshot(): Promise { + return this.capturePageState({ includeScreenshot: true }); + } + + async executeAction(code: string): Promise { + return this.runWithBrowserRecovery('executeAction', () => this.createAction().execute(code)); + } + + async attemptAction(code: string, originalMessage?: string, experience = true): Promise { + return this.runWithBrowserRecovery('attemptAction', () => this.createAction().attempt(code, originalMessage, experience)); + } + getPlaywrightRecorder(): PlaywrightRecorder { return this.playwrightRecorder; } async visit(url: string) { + return this.runWithBrowserRecovery('visit', () => this.visitOnce(url)); + } + + private async visitOnce(url: string) { await this.closeOtherTabs(); const serializedUrl = JSON.stringify(url); @@ -346,12 +400,14 @@ class Explorer { } async annotateElements(): Promise { - const { elements } = await annotatePageElements(this.playwrightHelper.page); - return elements; + return this.runWithBrowserRecovery('annotateElements', async () => { + const { elements } = await annotatePageElements(this.playwrightHelper.page); + return elements; + }); } async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise { - return visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || []); + return this.runWithBrowserRecovery('visuallyAnnotateElements', () => visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || [])); } async getEidxInContainer(containerCss: string | null): Promise { @@ -411,8 +467,7 @@ class Explorer { } isFatalBrowserError(error: unknown): boolean { - const msg = error instanceof Error ? error.message : String(error); - return FATAL_BROWSER_ERRORS.test(msg); + return isFatalBrowserError(error); } async recoverFromBrowserError(): Promise { @@ -422,7 +477,7 @@ class Explorer { if (!context) return await this.restartBrowser(); const page = await context.newPage(); await page.bringToFront(); - this.playwrightHelper.page = page; + await this.playwrightHelper._setPage(page); this.bindFrameNavigated(page); if (this.xhrCapture) { this.xhrCapture.attach(this.playwrightHelper.page); @@ -433,11 +488,11 @@ class Explorer { if (url) { tag('warning').log(`Browser error detected, recovering by navigating to ${url}`); await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); - return true; + return this.waitForUsablePageDom(); } tag('warning').log('Browser error detected, reloading page'); await this.playwrightHelper.page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 }); - return true; + return this.waitForUsablePageDom(); } catch (err) { tag('error').log(`Browser recovery failed: ${err instanceof Error ? err.message : err}`); return false; @@ -477,6 +532,7 @@ class Explorer { if (url) { await this.playwrightHelper.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 10000 }); + if (!(await this.waitForUsablePageDom())) return false; } tag('success').log('Browser restarted'); @@ -494,6 +550,30 @@ class Explorer { } } + private async waitForUsablePageDom(): Promise { + const page = this.playwrightHelper?.page; + if (!page) return false; + + await page.waitForLoadState?.('domcontentloaded', { timeout: 5000 }).catch(() => {}); + if (page.waitForFunction) { + const hasUsableDom = await page + .waitForFunction( + () => { + const body = document.body; + if (!body) return false; + return body.children.length > 0 || body.textContent?.trim().length > 0; + }, + undefined, + { timeout: 5000 } + ) + .then(() => true) + .catch(() => false); + if (!hasUsableDom) return false; + } + await page.waitForLoadState?.('networkidle', { timeout: 3000 }).catch(() => {}); + return true; + } + async isInsideIframe(): Promise { if (this.playwrightHelper.frame) return true; @@ -623,10 +703,11 @@ class Explorer { async startTest(test: Test): Promise { this._activeTest = test; + test.start(); await this.reporter.reportTestStart(test); await this.closeOtherTabs(); this.otherTabs = []; - if (!(await this.ensureActiveTestPageAvailable())) return false; + if (!(await this.ensurePageAvailable())) return false; const codeceptjsTest = toCodeceptjsTest(test); @@ -660,7 +741,7 @@ class Explorer { return true; } - async ensureActiveTestPageAvailable(): Promise { + async ensurePageAvailable(): Promise { const page = this.playwrightHelper?.page; if (page && !page.isClosed?.()) { this.watchActiveTestPage(page); @@ -673,6 +754,60 @@ class Explorer { return true; } + async ensureActiveTestPageAvailable(): Promise { + return this.ensurePageAvailable(); + } + + async handleExecutionError(error: unknown): Promise { + const message = error instanceof Error ? error.message : String(error); + tag('error').log(`Browser execution error: ${message}`); + + if (error instanceof Error && error.name === 'AbortError') { + return { + action: 'stop', + message, + }; + } + + if (error instanceof BrowserRecoveryError) { + return { + action: 'stop', + recovered: error.recovered, + message: error.message, + }; + } + + if (!this.isFatalBrowserError(error)) { + return { + action: 'continue', + message: `Previous execution error: ${message}. Investigate the current state and choose a different approach.`, + }; + } + + let recovered = await this.recoverFromBrowserError(); + if (!recovered) recovered = await this.restartBrowser(); + + if (!recovered) { + return { + action: 'stop', + recovered: false, + message: `Browser could not be recovered after fatal error: ${message}`, + }; + } + + this.watchActiveTestPage(); + return { + action: 'continue', + recovered: true, + message: dedent` + Browser was recovered after a fatal page error. + Continue from the restored page. + The interrupted browser action is not product evidence. + Inspect the restored page and retry the current step when it is still required. + `, + }; + } + watchActiveTestPage(page = this.playwrightHelper?.page): void { if (!this._activeTest) return; if (!page) return; @@ -794,7 +929,7 @@ class Explorer { await oldPage.close(); await newPage.bringToFront(); - this.playwrightHelper.page = newPage; + await this.playwrightHelper._setPage(newPage); this.otherTabs = []; this.bindFrameNavigated(newPage); @@ -828,7 +963,7 @@ class Explorer { } await firstPage.bringToFront(); - this.playwrightHelper.page = firstPage; + await this.playwrightHelper._setPage(firstPage); debugLog(`Cleaned up tabs, now focused on: ${await firstPage.url()}`); } diff --git a/src/utils/browser-errors.ts b/src/utils/browser-errors.ts new file mode 100644 index 0000000..1966722 --- /dev/null +++ b/src/utils/browser-errors.ts @@ -0,0 +1,25 @@ +// Playwright and CodeceptJS surface browser/page disposal as plain Error objects, +// not typed exceptions. Keep those external message markers in one adapter so +// recovery decisions are not duplicated across agents/actions. +const FATAL_BROWSER_ERROR_MARKERS = ['Frame was detached', 'Target closed', 'Target page, context or browser has been closed', 'Execution context was destroyed', 'Protocol error', 'Session closed']; + +export class BrowserRecoveryError extends Error { + constructor( + label: string, + public originalError: unknown, + public recovered: boolean + ) { + super(`${label} failed ${recovered ? 'after browser recovery' : 'because browser could not be recovered'}: ${browserErrorMessage(originalError)}`); + this.name = 'BrowserRecoveryError'; + } +} + +export function isFatalBrowserError(error: unknown): boolean { + if (error instanceof BrowserRecoveryError) return true; + const message = (error instanceof Error ? error.message : String(error)).toLowerCase(); + return FATAL_BROWSER_ERROR_MARKERS.some((marker) => message.includes(marker.toLowerCase())); +} + +export function browserErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/tests/integration/researcher-browser.test.ts b/tests/integration/researcher-browser.test.ts index 3af2d9e..b4d9a8c 100644 --- a/tests/integration/researcher-browser.test.ts +++ b/tests/integration/researcher-browser.test.ts @@ -98,6 +98,9 @@ describe('Researcher with real browser + aimock', () => { getConfig: () => ConfigParser.getInstance().getConfig(), visit: async () => {}, annotateElements: async () => (await annotatePageElements(page)).elements, + capturePageState: async () => ActionResult.fromState(state), + capturePageWithScreenshot: async () => ActionResult.fromState(state), + runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), createAction: () => ({ capturePageState: async () => ActionResult.fromState(state), caputrePageWithScreenshot: async () => ActionResult.fromState(state), diff --git a/tests/integration/researcher-sections.test.ts b/tests/integration/researcher-sections.test.ts index c1ac258..bf0eece 100644 --- a/tests/integration/researcher-sections.test.ts +++ b/tests/integration/researcher-sections.test.ts @@ -42,6 +42,9 @@ function createMockExplorer(configOverrides: Record = {}, playw getConfig: () => config, visit: async () => {}, annotateElements: async () => [], + capturePageState: async () => ActionResult.fromState(fakeState), + capturePageWithScreenshot: async () => ActionResult.fromState(fakeState), + runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), createAction: () => ({ capturePageState: async () => ActionResult.fromState(fakeState), caputrePageWithScreenshot: async () => ActionResult.fromState(fakeState), diff --git a/tests/integration/researcher.test.ts b/tests/integration/researcher.test.ts index fc178ad..79da34b 100644 --- a/tests/integration/researcher.test.ts +++ b/tests/integration/researcher.test.ts @@ -68,6 +68,9 @@ function createMockExplorer(state = fakeState) { getConfig: () => ConfigParser.getInstance().getConfig(), visit: async () => {}, annotateElements: async () => [], + capturePageState: async () => ActionResult.fromState(state), + capturePageWithScreenshot: async () => ActionResult.fromState(state), + runWithBrowserRecovery: async (_label: string, operation: () => Promise) => operation(), createAction: () => ({ capturePageState: async () => ActionResult.fromState(state), caputrePageWithScreenshot: async () => ActionResult.fromState(state), diff --git a/tests/unit/captain-artifacts.test.ts b/tests/unit/captain-artifacts.test.ts index a529354..fa7f068 100644 --- a/tests/unit/captain-artifacts.test.ts +++ b/tests/unit/captain-artifacts.test.ts @@ -32,16 +32,36 @@ describe('Captain artifact analysis tools', () => { expect(notes).toEqual(['baseUrl: https://example.test\nbrowser: chromium', 'Displayed config details']); }); - it('rejects informational requests completed without details', async () => { + it('rejects completion without details before any successful action', async () => { const captain = buildCaptain(); - const tools = (captain as any).coreTools(task('explain what page I am on'), () => {}); + const tools = (captain as any).coreTools(task('inspect the current state'), () => {}); - const result = await tools.done.execute({ summary: 'Explained current page' }); + const result = await tools.done.execute({ summary: 'Inspected current state' }); expect(result.success).toBe(false); expect(result.message).toContain('actual answer in details'); }); + it('allows completion without details after successful browser evidence', async () => { + const captain = buildCaptain(); + (captain as any).recentToolCalls = [{ wasSuccessful: true, output: { code: 'I.click("Submit")' } }]; + const tools = (captain as any).coreTools(task('click the submit button'), () => {}); + + const result = await tools.done.execute({ summary: 'Clicked submit' }); + + expect(result.success).toBe(true); + }); + + it('rejects completion without details after read-only tool output', async () => { + const captain = buildCaptain(); + (captain as any).recentToolCalls = [{ wasSuccessful: true, output: { content: 'report details' } }]; + const tools = (captain as any).coreTools(task('show report'), () => {}); + + const result = await tools.done.execute({ summary: 'Read report' }); + + expect(result.success).toBe(false); + }); + it('reads explicit report artifact paths without shell commands', async () => { ConfigParser.resetForTesting(); ConfigParser.setupTestConfig(); @@ -140,26 +160,26 @@ describe('Captain artifact analysis tools', () => { }); describe('Captain command guard', () => { - it('blocks test execution commands for natural-language analysis requests', async () => { + it('blocks slash commands for natural-language analysis requests', async () => { let called = false; const captain = buildCaptain(async () => { called = true; }); const tools = (captain as any).coreTools(task('analyze the latest report'), () => {}); - const result = await tools.runCommand.execute({ command: '/test failing_demo_for_captain_tui_explanation' }); + const result = await tools.runCommand.execute({ command: '/anything' }); expect(result.success).toBe(false); expect(called).toBe(false); expect(result.message).toContain('Command blocked'); }); - it('allows execution commands when the user explicitly typed that slash command', async () => { + it('allows slash commands when the user explicitly typed that slash command', async () => { let called = false; const captain = buildCaptain(async () => { called = true; }); - const tools = (captain as any).coreTools(task('/test 1'), () => {}); - const result = await tools.runCommand.execute({ command: '/test 1' }); + const tools = (captain as any).coreTools(task('/anything value'), () => {}); + const result = await tools.runCommand.execute({ command: '/anything value' }); expect(result.success).toBe(true); expect(called).toBe(true); diff --git a/tests/unit/captain-mode.test.ts b/tests/unit/captain-mode.test.ts index 45a45cf..d5e37cf 100644 --- a/tests/unit/captain-mode.test.ts +++ b/tests/unit/captain-mode.test.ts @@ -64,22 +64,27 @@ describe('Captain modes', () => { describe('Captain execution recovery', () => { it('continues after a fatal browser error is recovered', async () => { const captain = buildCaptainWithExplorer({ - isFatalBrowserError: () => true, - recoverFromBrowserError: async () => true, + handleExecutionError: async () => ({ + action: 'continue', + recovered: true, + message: 'Browser was recovered after a fatal page error.', + }), }); const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); expect(recovery.action).toBe('continue'); expect(recovery.recovered).toBe(true); - expect(recovery.message).toContain('Captain recovered the browser'); + expect(recovery.message).toContain('Browser was recovered'); }); it('stops when a fatal browser error cannot be recovered', async () => { const captain = buildCaptainWithExplorer({ - isFatalBrowserError: () => true, - recoverFromBrowserError: async () => false, - restartBrowser: async () => false, + handleExecutionError: async () => ({ + action: 'stop', + recovered: false, + message: 'Browser could not be recovered', + }), }); const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); @@ -90,9 +95,11 @@ describe('Captain execution recovery', () => { it('continues when browser restart recovers after page recovery fails', async () => { const captain = buildCaptainWithExplorer({ - isFatalBrowserError: () => true, - recoverFromBrowserError: async () => false, - restartBrowser: async () => true, + handleExecutionError: async () => ({ + action: 'continue', + recovered: true, + message: 'Browser was recovered after a fatal page error.', + }), }); const recovery = await captain.processExecutionError(new Error('Target closed'), { scenario: 'create project' } as any); @@ -103,8 +110,10 @@ describe('Captain execution recovery', () => { it('continues with guidance for non-fatal execution errors', async () => { const captain = buildCaptainWithExplorer({ - isFatalBrowserError: () => false, - recoverFromBrowserError: async () => false, + handleExecutionError: async () => ({ + action: 'continue', + message: 'Previous execution error: Locator not found. Investigate the current state and choose a different approach.', + }), }); const recovery = await captain.processExecutionError(new Error('Locator not found'), { scenario: 'create project' } as any); diff --git a/tests/unit/explorer-recovery-url.test.ts b/tests/unit/explorer-recovery-url.test.ts index f0b7002..d91c0be 100644 --- a/tests/unit/explorer-recovery-url.test.ts +++ b/tests/unit/explorer-recovery-url.test.ts @@ -40,6 +40,9 @@ describe('Explorer recovery URL resolution', () => { }; (explorer as any).playwrightHelper = { page: { isClosed: () => true }, + _setPage: async (page: any) => { + (explorer as any).playwrightHelper.page = page; + }, browserContext: { newPage: async () => newPage, }, @@ -56,4 +59,96 @@ describe('Explorer recovery URL resolution', () => { expect(navigated).toEqual(['https://the-internet.herokuapp.com/']); expect(boundEvents).toContain('framenavigated'); }); + + it('recovers and retries browser operations in Explorer', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + let attempts = 0; + let recoveries = 0; + (explorer as any).playwrightHelper = { + page: { isClosed: () => false }, + }; + (explorer as any).recoverFromBrowserError = async () => { + recoveries++; + return true; + }; + + const result = await explorer.runWithBrowserRecovery('test operation', async () => { + attempts++; + if (attempts === 1) throw new Error('Target closed'); + return 'recovered'; + }); + + expect(result).toBe('recovered'); + expect(attempts).toBe(2); + expect(recoveries).toBe(1); + }); + + it('recovers and retries action attempts through Explorer', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + let attempts = 0; + let recoveries = 0; + (explorer as any).playwrightHelper = { + page: { isClosed: () => false }, + }; + (explorer as any).recoverFromBrowserError = async () => { + recoveries++; + return true; + }; + (explorer as any).createAction = () => ({ + attempt: async () => { + attempts++; + if (attempts === 1) throw new Error('Target page, context or browser has been closed'); + return true; + }, + }); + + const result = await explorer.attemptAction('I.click("Menu")', undefined, false); + + expect(result).toBe(true); + expect(attempts).toBe(2); + expect(recoveries).toBe(1); + }); + + it('stops when an operation fails again after browser recovery', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + (explorer as any).playwrightHelper = { + page: { isClosed: () => false }, + }; + (explorer as any).recoverFromBrowserError = async () => true; + + let error: unknown; + try { + await explorer.runWithBrowserRecovery('capturePageState', async () => { + throw new Error('Target page, context or browser has been closed'); + }); + } catch (err) { + error = err; + } + + const result = await explorer.handleExecutionError(error); + + expect(result.action).toBe('stop'); + expect(result.message).toContain('failed after browser recovery'); + }); + + it('returns a stop decision when browser recovery fails', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + (explorer as any).recoverFromBrowserError = async () => false; + (explorer as any).restartBrowser = async () => false; + + const result = await explorer.handleExecutionError(new Error('Target closed')); + + expect(result.action).toBe('stop'); + expect(result.recovered).toBe(false); + }); + + it('returns guidance for non-browser execution errors', async () => { + const explorer = buildExplorer('https://the-internet.herokuapp.com'); + + const result = await explorer.handleExecutionError(new Error('Locator not found')); + + expect(result.action).toBe('continue'); + expect(result.recovered).toBeUndefined(); + expect(result.message).toContain('Previous execution error'); + }); }); diff --git a/tests/unit/tester-execution-recovery.test.ts b/tests/unit/tester-execution-recovery.test.ts deleted file mode 100644 index 9a324e4..0000000 --- a/tests/unit/tester-execution-recovery.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import { Tester } from '../../src/ai/tester.ts'; -import { TestResult } from '../../src/test-plan.ts'; - -function buildTester(captain?: any, page: any = { id: 'recovered-page' }, explorerOverrides: Record = {}): Tester { - const explorer: any = { - getConfig: () => ({}), - playwrightHelper: { - page, - }, - ensureActiveTestPageAvailable: async () => !!page && !page.isClosed?.(), - watchActiveTestPage: () => {}, - createAction: () => ({ - capturePageState: async () => ({ - url: '/', - title: 'Recovered', - hash: 'recovered', - ariaSnapshot: '', - getInteractiveARIA: () => '', - isInsideIframe: false, - }), - }), - hasOtherTabs: () => false, - getCurrentIframeInfo: () => null, - stopTest: async () => {}, - ...explorerOverrides, - }; - const researcher = { - research: async () => '', - }; - const tester = new Tester(explorer, {} as any, researcher as any, {} as any); - if (captain) tester.setCaptain(captain); - return tester; -} - -function buildTask() { - const notes: string[] = []; - return { - hasFinished: false, - result: null, - addNote: (message: string) => notes.push(message), - finish(result: any) { - this.hasFinished = true; - this.result = result; - }, - get isSuccessful() { - return this.result === TestResult.PASSED; - }, - get isSkipped() { - return this.result === TestResult.SKIPPED; - }, - get hasFailed() { - return this.result === TestResult.FAILED; - }, - notes, - scenario: 'startup recovery test', - }; -} - -function buildConversation() { - const messages: string[] = []; - return { - addUserText: (message: string) => messages.push(message), - messages, - }; -} - -describe('Tester execution recovery', () => { - it('continues after Captain recovers the browser', async () => { - const captain = { - processExecutionError: async () => ({ - action: 'continue', - recovered: true, - message: 'Recovered browser, continue from restored page', - }), - }; - const recoveredPage = { id: 'recovered-page' }; - const tester = buildTester(captain, { isClosed: () => true }); - (tester as any).explorer.playwrightHelper.page = recoveredPage; - const task = buildTask(); - const conversation = buildConversation(); - const watchedPages: any[] = []; - (tester as any).explorer.watchActiveTestPage = () => watchedPages.push((tester as any).explorer.playwrightHelper.page); - let stopped = false; - - await (tester as any).handleExecutionError(task, conversation, new Error('Target closed'), () => { - stopped = true; - }); - - expect(stopped).toBe(false); - expect(task.hasFinished).toBe(false); - expect(watchedPages).toHaveLength(1); - expect(conversation.messages[0]).toContain('Recovered browser'); - expect(conversation.messages[0]).toContain(''); - }); - - it('stops the test when Captain cannot recover', async () => { - const captain = { - processExecutionError: async () => ({ - action: 'stop', - recovered: false, - message: 'Recovery failed', - }), - }; - const tester = buildTester(captain); - const task = buildTask(); - const conversation = buildConversation(); - let stopped = false; - - await (tester as any).handleExecutionError(task, conversation, new Error('Target closed'), () => { - stopped = true; - }); - - expect(stopped).toBe(true); - expect(task.hasFinished).toBe(true); - expect(task.result).toBe(TestResult.FAILED); - expect(conversation.messages).toHaveLength(0); - }); - - it('falls back to retry guidance when Captain is unavailable', async () => { - const tester = buildTester(); - const task = buildTask(); - const conversation = buildConversation(); - let stopped = false; - - await (tester as any).handleExecutionError(task, conversation, new Error('Locator not found'), () => { - stopped = true; - }); - - expect(stopped).toBe(false); - expect(conversation.messages[0]).toContain('Previous AI call failed'); - }); - - it('recovers when the browser page is already closed before the next step', async () => { - const tester = buildTester(undefined, { isClosed: () => true }); - const recoveredPage = { id: 'recovered-page' }; - const captain = { - processExecutionError: async () => { - (tester as any).explorer.playwrightHelper.page = recoveredPage; - return { - action: 'continue', - recovered: true, - message: 'Recovered closed page', - }; - }, - }; - tester.setCaptain(captain as any); - const task = buildTask(); - const conversation = buildConversation(); - const watchedPages: any[] = []; - (tester as any).explorer.watchActiveTestPage = () => watchedPages.push((tester as any).explorer.playwrightHelper.page); - let stopped = false; - - const available = await (tester as any).ensureBrowserPageAvailable(task, conversation, () => { - stopped = true; - }); - - expect(available).toBe(true); - expect(stopped).toBe(false); - expect(watchedPages).toHaveLength(1); - expect(conversation.messages[0]).toContain('Recovered closed page'); - expect(conversation.messages[0]).toContain(''); - }); - - it('retries initial navigation after Captain recovers the browser', async () => { - let visits = 0; - const tester = buildTester( - undefined, - { isClosed: () => false }, - { - visit: async () => { - visits++; - if (visits === 1) throw new Error('Cannot navigate: page has been closed'); - }, - isFatalBrowserError: () => true, - } - ); - const captain = { - processExecutionError: async () => ({ - action: 'continue', - recovered: true, - message: 'Recovered before initial navigation', - }), - }; - tester.setCaptain(captain as any); - const task = buildTask(); - task.startUrl = '/'; - const conversation = buildConversation(); - - const navigated = await (tester as any).visitStartUrlWithRecovery(task, conversation); - - expect(navigated).toBe(true); - expect(visits).toBe(2); - expect(task.hasFinished).toBe(false); - }); - - it('cleans up started test lifecycle on early startup failure', async () => { - let stopped = false; - const tester = buildTester( - undefined, - { isClosed: () => false }, - { - stopTest: async () => { - stopped = true; - }, - } - ); - const task = buildTask(); - task.startUrl = '/'; - - await (tester as any).cleanupStartedTest(task); - - expect(stopped).toBe(true); - }); -}); From 5a1fd8df317334d6f263c6bcfc1a384d033127b4 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Tue, 9 Jun 2026 16:53:11 +0300 Subject: [PATCH 6/6] fix test --- src/action.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/action.ts b/src/action.ts index 31a32f3..09d35b6 100644 --- a/src/action.ts +++ b/src/action.ts @@ -79,7 +79,7 @@ class Action { const frame = this.playwrightHelper.frame; await page?.waitForLoadState('domcontentloaded', { timeout: 10000 })?.catch(() => {}); await waitForUsablePageDom(page); - const grabAll = () => Promise.all([captureHtml(page, frame), captureTitle(page), this.captureBrowserLogs()]); + const grabAll = () => Promise.all([captureHtml(page, frame, this.actor), captureTitle(page, this.actor), this.captureBrowserLogs()]); const [html, title, browserLogs] = await grabAll().catch(async (err: Error) => { const msg = err instanceof Error ? err.message : String(err); if (!/navigating and changing the content/i.test(msg)) throw err; @@ -426,14 +426,16 @@ async function waitForUsablePageDom(page: any): Promise { .catch(() => {}); } -async function captureHtml(page: any, frame: any): Promise { +async function captureHtml(page: any, frame: any, actor: any): Promise { if (frame?.content) return frame.content(); if (page?.content) return page.content(); + if (actor?.grabSource) return actor.grabSource(); throw new Error('Playwright page is unavailable for HTML capture'); } -async function captureTitle(page: any): Promise { +async function captureTitle(page: any, actor: any): Promise { if (page?.title) return page.title(); + if (actor?.grabTitle) return actor.grabTitle(); return ''; }