diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef697d..fd089f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 2026-06-06 ### Changes +- [Researcher] Deep research (`--deep`) now builds on previous runs instead of starting over. It reloads the hidden sections (dropdowns, modals, expanded panels, tabs) found last time — even in an earlier session — and replays the action that revealed each to confirm it still works. Sections that still open are reused as-is, ones whose button moved or was renamed are flagged so the AI looks for them again, and when everything still matches the slow click-through exploration is skipped entirely. Previously every deep run re-discovered hidden UI from scratch and could silently miss sections it had found before. - [Tester] Checkboxes and other toggle controls (switches, selectable rows/items) are no longer flipped back off by accident. The Tester now sets a checkbox with the idempotent `I.checkOption` / `I.uncheckOption` commands instead of `I.click`, so selecting an already-selected checkbox keeps it selected. It also reads the page state after each click and stops re-clicking a control once it already shows the wanted result (checked, selected, or a count/label confirming success). Previously a second click on a selected checkbox could toggle it off — for example dropping a "Select 32 tests" selection back to "Select 0 tests" and saving an empty result. - [Tester] Before filling a form that saves data (create/update), the Tester now reads each field's requirements — required, type/format, length, and placeholder/hint text — and enters values that satisfy them, instead of submitting and discovering validation errors. Search, filter, and sort forms that only change the view are skipped. diff --git a/docs/researcher.md b/docs/researcher.md index 24c3cc6..7f62660 100644 --- a/docs/researcher.md +++ b/docs/researcher.md @@ -156,7 +156,7 @@ Analyzes page using HTML and ARIA tree. Fast and works with any model. /research --deep ``` -Expands hidden elements (dropdowns, accordions, tabs) to discover more UI. Clicks through interactive elements and documents what appears. +Expands hidden elements (dropdowns, accordions, tabs) to discover more UI. Clicks through interactive elements and documents what appears. Deep research also reuses what it found on previous runs — see [Reusing Previous Results](#reusing-previous-results). #### Research with Data Extraction @@ -225,6 +225,17 @@ For each element, the researcher: 3. Detects what changed (navigation, modal, menu, UI change) 4. Restores original state (Escape key or navigate back) +### Reusing Previous Results + +Hidden sections discovered by deep research are saved under an **Extended Research** block in the page's research file, together with the action that revealed each one. On the next deep run for the same page — even in a later session — the researcher builds on that instead of starting from scratch: + +1. **Replay** — it re-runs the saved action for every previously found section to check it still opens. +2. **Reuse** — sections that still open are kept as-is and are not explored again, so the run spends its click budget on what is actually new. +3. **Re-discover** — if a section's trigger no longer works (the button moved or was renamed), it is flagged to the AI as "this section existed before, find it again", so a relocated control is recovered rather than lost. +4. **Skip** — when every known section still opens and the click budget is already covered, the slow click-through exploration is skipped because the page is effectively unchanged. + +This makes repeated deep runs faster and stops the researcher from silently losing hidden UI it had already mapped. The reuse reads the last saved research file directly, so it works across sessions and is not limited by the in-memory [cache window](#caching). + ### Filtering Elements Not all elements should be explored. The researcher filters by: @@ -313,6 +324,8 @@ Use `--force` to bypass cache: /research --force ``` +This cache controls when a fresh result is reused within a session. It is separate from how [deep research reuses previous results](#reusing-previous-results): a deep run always reloads the last saved research file from `output/research/` to replay and verify previously discovered hidden sections, regardless of the cache window or session. + ## Configuration Examples ### Skip Cookie Banners and Ads diff --git a/src/ai/researcher/cache.ts b/src/ai/researcher/cache.ts index b2fb71b..dab3861 100644 --- a/src/ai/researcher/cache.ts +++ b/src/ai/researcher/cache.ts @@ -49,6 +49,13 @@ export function getCachedResearch(hash: string): string { return cached; } +export function getPreviousResearch(hash: string): string { + if (!hash) return ''; + const researchFile = outputPath('research', `${hash}.md`); + if (!existsSync(researchFile)) return ''; + return readFileSync(researchFile, 'utf8'); +} + export function saveResearch(hash: string, text: string, combinedHtml?: string): string { const researchDir = outputPath('research'); const researchFile = join(researchDir, `${hash}.md`); diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index 1f7296b..b8cf8bc 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -5,10 +5,11 @@ import type Explorer from '../../explorer.ts'; import type { StateManager } from '../../state-manager.js'; import { WebPageState } from '../../state-manager.js'; import { detectFocusArea, diffAriaSnapshots } from '../../utils/aria.ts'; +import { extractCodeBlocks } from '../../utils/code-extractor.ts'; import { tag } from '../../utils/logger.js'; import { mdq } from '../../utils/markdown-query.ts'; import type { Provider } from '../provider.js'; -import { getCachedResearch, saveResearch } from './cache.ts'; +import { getCachedResearch, getPreviousResearch, saveResearch } from './cache.ts'; import { type Constructor, debugLog } from './mixin.ts'; import { type ResearchElement, parseResearchSections } from './parser.ts'; import type { ResearchResult } from './research-result.ts'; @@ -26,14 +27,30 @@ export function WithDeepAnalysis(Base: T) { tag('info').log('Starting deep analysis of expandable elements'); await (this as any).navigateTo(state.fullUrl || state.url); - let expandables = await this._discoverExpandables(result.text); - if (expandables.length === 0) { - tag('info').log('No expandable elements identified by AI'); - return; + const maxClicks = (this.explorer.getConfig().ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; + + const expandedSections: string[] = []; + const navigationLinks: Array<{ code: string; url: string }> = []; + let verifiedCodes: string[] = []; + let missing: PreviousSection[] = []; + + const previousSections = this._loadPreviousExtendedSections(state.hash || ''); + if (previousSections.length > 0) { + tag('substep').log(`Replaying ${previousSections.length} previously discovered sections`); + const replay = await this._replayPreviousSections(state, previousSections, maxClicks); + expandedSections.push(...replay.verified); + verifiedCodes = replay.verifiedCodes; + missing = replay.missing; + tag('info').log(`Reused ${replay.verified.length}/${previousSections.length} previous sections, ${missing.length} to re-discover`); + + if (missing.length === 0 && replay.verified.length >= maxClicks) { + tag('info').log('Page appears unchanged, reusing previous sections and skipping discovery'); + this._appendExtendedResearch(result, expandedSections, navigationLinks); + return; + } } - tag('substep').log(`Identified ${expandables.length} expandable elements`); - const maxClicks = (this.explorer.getConfig().ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS; + let expandables = await this._discoverExpandables(result.text, missing, verifiedCodes); if (expandables.length > maxClicks) { expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks); tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`); @@ -44,10 +61,12 @@ export function WithDeepAnalysis(Base: T) { commands: this._buildClickCommands(el), description: el.name, })) - .filter((el) => el.commands.length > 0); + .filter((el) => el.commands.length > 0) + .filter((el) => !el.commands.some((cmd) => verifiedCodes.includes(cmd))); if (elements.length === 0) { - tag('info').log('No expandables with valid locators'); + tag('info').log('No new expandable elements to click'); + this._appendExtendedResearch(result, expandedSections, navigationLinks); return; } @@ -57,25 +76,11 @@ export function WithDeepAnalysis(Base: T) { for (const el of elements) debugLog(`Expandable: ${el.description} → ${el.commands[0]}`); tag('substep').log(`Clicking ${elements.length} expandable elements`); - const expandedSections: string[] = []; - const navigationLinks: Array<{ code: string; url: string }> = []; - await this._clickExpandableElements(elements, state, expandedSections, navigationLinks); tag('info').log(`Deep analysis complete. Sections: ${expandedSections.length}, navigation links: ${navigationLinks.length}`); - const dedupedSections = this._deduplicateExpandedSections(expandedSections); - if (dedupedSections.length !== expandedSections.length) { - tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`); - } - - if (dedupedSections.length > 0) { - result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`; - } - if (navigationLinks.length > 0) { - const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n'); - result.text += `\n\n## Navigation Links\n\n${links}`; - } + this._appendExtendedResearch(result, expandedSections, navigationLinks); } async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise { @@ -127,7 +132,70 @@ export function WithDeepAnalysis(Base: T) { return sectionMarkdown; } - private async _discoverExpandables(researchText: string): Promise { + private _loadPreviousExtendedSections(hash: string): PreviousSection[] { + if (!hash) return []; + const previous = getPreviousResearch(hash); + if (!previous) return []; + + const sections: PreviousSection[] = []; + for (const section of parseResearchSections(previous)) { + if (!section.isExtended) continue; + const code = extractCodeBlocks(section.rawMarkdown)[0]; + if (!code) continue; + sections.push({ name: section.name, code }); + } + return sections; + } + + private async _replayPreviousSections(state: WebPageState, prevSections: PreviousSection[], maxClicks: number): Promise<{ verified: string[]; verifiedCodes: string[]; missing: PreviousSection[] }> { + const originalAria = state.ariaSnapshot || ''; + const verified: string[] = []; + const verifiedCodes: string[] = []; + const missing: PreviousSection[] = []; + + for (const section of prevSections.slice(0, maxClicks)) { + if (executionController.isInterrupted()) break; + + let outcome: ExpansionOutcome; + try { + outcome = await this._executeAndAnalyze([section.code], section.name, state, originalAria, this._summarizeExpanded(verified)); + } catch (err) { + tag('warning').log(`Replay failed for "${section.name}": ${err instanceof Error ? err.message : err}`); + await this._restorePageState(state.url, originalAria).catch(() => {}); + missing.push(section); + continue; + } + + if (outcome.status === 'revealed') { + verified.push(outcome.sectionMarkdown); + verifiedCodes.push(section.code); + debugLog(`Replayed and verified section: ${section.name}`); + continue; + } + + debugLog(`Could not replay previous section: ${section.name}`); + missing.push(section); + } + + return { verified, verifiedCodes, missing }; + } + + private _appendExtendedResearch(result: ResearchResult, expandedSections: string[], navigationLinks: Array<{ code: string; url: string }>): void { + const dedupedSections = this._deduplicateExpandedSections(expandedSections); + if (dedupedSections.length !== expandedSections.length) { + tag('substep').log(`Deduplicated ${expandedSections.length} → ${dedupedSections.length} extended sections`); + } + + if (dedupedSections.length > 0) { + result.text += `\n\n# Extended Research\n\n${dedupedSections.join('\n\n---\n\n')}`; + } + if (navigationLinks.length > 0) { + const links = navigationLinks.map((l) => `- \`${l.code}\` opens ${l.url}`).join('\n'); + result.text += `\n\n## Navigation Links\n\n${links}`; + } + } + + private async _discoverExpandables(researchText: string, missing: PreviousSection[] = [], verifiedCodes: string[] = []): Promise { const allElements = new Map(); for (const section of parseResearchSections(researchText)) { for (const el of section.elements) { @@ -138,6 +206,16 @@ export function WithDeepAnalysis(Base: T) { const eidxList = [...allElements.keys()].join(', '); + let missingHint = ''; + if (missing.length > 0) { + const list = missing.map((s) => `- "${s.name}" (previously revealed via ${s.code})`).join('\n'); + missingHint = dedent` + + These sections were present on a previous visit but their trigger could not be replayed now — the element may have moved or been renamed. Prioritize finding the element that now reveals each: + ${list} + `; + } + const textPrompt = dedent` From this UI research, identify elements that could reveal hidden UI when clicked (dropdown menus, popups, expandable panels, accordion sections, overflow menus, tab switches). @@ -145,6 +223,7 @@ export function WithDeepAnalysis(Base: T) { Available eidx refs: ${eidxList} ${researchText} + ${missingHint} Rules: - Only pick elements that HIDE content until clicked (menus, dropdowns, accordions, tabs) @@ -168,6 +247,7 @@ export function WithDeepAnalysis(Base: T) { Look for: overflow/ellipsis menus, chevron dropdowns, hamburger menus, gear/settings buttons, accordion toggles, tab switches, filter buttons. + ${missingHint} Rules: - For repeated icons (same icon on every list row), pick only the FIRST one @@ -302,53 +382,15 @@ export function WithDeepAnalysis(Base: T) { } } - let clickCode: string | null = null; - const action = this.explorer.createAction(); - for (const cmd of el.commands) { - if (await action.attempt(cmd, undefined, false)) { - clickCode = cmd; - break; - } - } - if (!clickCode) { - debugLog(`Click failed: ${el.description.slice(0, 80)}`); - continue; - } - - await new Promise((r) => setTimeout(r, 500)); - - let diff: Diff; - try { - await this.explorer.createAction().capturePageState(); - const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!); - diff = await currAR.diff(previousState); - await diff.calculate(); - } catch (err) { - tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`); - await this._restorePageState(state.url, originalAria); - continue; - } - - if (diff.urlHasChanged()) { - debugLog(`Click navigated to ${this.stateManager.getCurrentState()?.url}`); - navigationLinks.push({ code: clickCode, url: this.stateManager.getCurrentState()?.url || '' }); - await (this as any).navigateTo(state.url); - continue; - } - - const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0); - if (!diff.ariaChanged && clickHtmlSize <= 150) { - debugLog(`No changes from: ${el.description.slice(0, 80)}`); + const outcome = await this._executeAndAnalyze(el.commands, el.description, state, originalAria, this._summarizeExpanded(expandedSections)); + if (outcome.status === 'navigated') { + navigationLinks.push({ code: outcome.code, url: outcome.url }); continue; } - - const sectionMarkdown = await this._analyzeExpandedAction(clickCode, el.description, diff, this._summarizeExpanded(expandedSections)); - if (sectionMarkdown) { - expandedSections.push(sectionMarkdown); + if (outcome.status === 'revealed') { + expandedSections.push(outcome.sectionMarkdown); debugLog(`Captured section from: ${el.description.slice(0, 80)}`); } - - await this._restorePageState(state.url, originalAria); } catch (err) { tag('warning').log(`Expandable click failed for "${el.description.slice(0, 80)}": ${err instanceof Error ? err.message : err}`); try { @@ -358,6 +400,55 @@ export function WithDeepAnalysis(Base: T) { } } + private async _executeAndAnalyze(commands: string[], description: string, state: WebPageState, originalAria: string, alreadyExpanded: string[]): Promise { + const previousState = ActionResult.fromState(this.stateManager.getCurrentState()!); + + let clickCode: string | null = null; + const action = this.explorer.createAction(); + for (const cmd of commands) { + if (await action.attempt(cmd, undefined, false)) { + clickCode = cmd; + break; + } + } + if (!clickCode) { + debugLog(`Click failed: ${description.slice(0, 80)}`); + return { status: 'failed' }; + } + + await new Promise((r) => setTimeout(r, 500)); + + let diff: Diff; + try { + await this.explorer.createAction().capturePageState(); + const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!); + diff = await currAR.diff(previousState); + await diff.calculate(); + } catch (err) { + tag('warning').log(`State capture failed after click: ${err instanceof Error ? err.message : err}`); + await this._restorePageState(state.url, originalAria); + return { status: 'failed' }; + } + + if (diff.urlHasChanged()) { + const url = this.stateManager.getCurrentState()?.url || ''; + debugLog(`Click navigated to ${url}`); + await (this as any).navigateTo(state.url); + return { status: 'navigated', code: clickCode, url }; + } + + const clickHtmlSize = diff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0); + if (!diff.ariaChanged && clickHtmlSize <= 150) { + debugLog(`No changes from: ${description.slice(0, 80)}`); + return { status: 'none', code: clickCode }; + } + + const sectionMarkdown = await this._analyzeExpandedAction(clickCode, description, diff, alreadyExpanded); + await this._restorePageState(state.url, originalAria); + if (!sectionMarkdown) return { status: 'none', code: clickCode }; + return { status: 'revealed', code: clickCode, sectionMarkdown }; + } + private async _restorePageState(url: string, originalAria: string): Promise { try { await (this as any).cancelInUi(); @@ -484,6 +575,13 @@ interface ExpandableElement extends ResearchElement { container: string | null; } +interface PreviousSection { + name: string; + code: string; +} + +type ExpansionOutcome = { status: 'revealed'; code: string; sectionMarkdown: string } | { status: 'navigated'; code: string; url: string } | { status: 'none'; code: string } | { status: 'failed' }; + export interface DeepAnalysisMethods { performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise; researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise; diff --git a/tests/unit/research-parser.test.ts b/tests/unit/research-parser.test.ts index 9cd9cb7..f16015e 100644 --- a/tests/unit/research-parser.test.ts +++ b/tests/unit/research-parser.test.ts @@ -1,6 +1,7 @@ import dedent from 'dedent'; import { describe, expect, it } from 'vitest'; import { formatResearchSummary, parseResearchSections } from '../../src/ai/researcher/parser.ts'; +import { extractCodeBlocks } from '../../src/utils/code-extractor.ts'; const researchMarkdown = dedent` ## Page Purpose @@ -87,6 +88,35 @@ describe('parseResearchSections', () => { const sections = parseResearchSections(md); expect(sections.every((s) => !s.isExtended)).toBe(true); }); + + it('keeps the replay action code in an extended section rawMarkdown', () => { + const md = dedent` + ## List + + Suite list content. + > Container: \`.suites-list-content\` + + # Extended Research + + ### Dropdown Expansion + + Action: + + \`\`\`js + I.click({ role: 'button', text: 'New' }, '.toolbar') + \`\`\` + + A dropdown menu appeared. + + | Element | Type | ARIA | CSS | + |------|------|------|------| + | Folder | button | { role: 'button', text: 'Folder' } | '.folder' | + `; + + const extended = parseResearchSections(md).filter((s) => s.isExtended); + expect(extended).toHaveLength(1); + expect(extractCodeBlocks(extended[0].rawMarkdown)[0]).toBe("I.click({ role: 'button', text: 'New' }, '.toolbar')"); + }); }); describe('formatResearchSummary', () => {