diff --git a/packages/hackbrowser/src/agent.ts b/packages/hackbrowser/src/agent.ts index b32be831e..86e3c9d4d 100644 --- a/packages/hackbrowser/src/agent.ts +++ b/packages/hackbrowser/src/agent.ts @@ -198,7 +198,7 @@ function waitForBrowserClose(browser: import("playwright").Browser, signal?: Abo /** * Mark that login was detected. The actual re-queue happens in the BFS loop * AFTER the current page finishes exploration (so new discoveries from - * collectDOMLinks are enqueued first, before re-visit URLs). + * collectNavLinks are enqueued first, before re-visit URLs). */ function triggerReDiscovery(globalState: ReturnType): void { if (globalState.authPhase === "authenticated") return @@ -1084,12 +1084,12 @@ async function explorePageWithAI( // 6. Collect same-host links from DOM (BFS supplement) try { - const domLinks = await collectDOMLinks(page, pageUrl, inScope) + const domLinks = await collectNavLinks(page, pageUrl, inScope) for (const url of domLinks) { if (!linksToEnqueue.includes(url)) linksToEnqueue.push(url) } } catch { - log.debug("collectDOMLinks failed (page may have navigated)") + log.debug("collectNavLinks failed (page may have navigated)") } return linksToEnqueue @@ -1743,27 +1743,52 @@ function resolveUrl(href: string, baseUrl: string, inScope: ScopeMatcher): strin return null } -/** Collect links from DOM as BFS supplement. */ -async function collectDOMLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise { - const hrefs: string[] = await page.$$eval("a[href]", (els) => - els.map((el) => (el as HTMLAnchorElement).href).filter(Boolean), +// Attributes that DECLARATIVELY encode a navigation destination readable +// without clicking (a real URL, path, or hash-route). Order = priority per +// element. `routerLink`/`[to]`/`data-route` are intentionally excluded — their +// values are router-relative and need scheme-guessing; those routes are +// discovered imperatively by clicking (phase B, #120). +const DECLARATIVE_NAV_ATTRS = ["href", "data-href", "data-url"] as const +const NAV_TARGET_SELECTOR = "a[href], [role=link], [data-href], [data-url]" + +/** Collect declarative navigation links from DOM as a BFS supplement. */ +async function collectNavLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise { + // Beyond : many SPAs navigate via non-anchor elements that still + // declare their destination in an attribute (data-href/data-url) or carry + // role=link. Harvest the first destination-bearing attribute per element; the + // raw value is resolved against the page URL below (so "/x" and "#/x" work). + const hrefs: string[] = await page.$$eval( + NAV_TARGET_SELECTOR, + (els, attrs) => { + const out: string[] = [] + for (const el of els) { + for (const attr of attrs) { + const value = el.getAttribute(attr) + if (value) { + out.push(value) + break + } + } + } + return out + }, + [...DECLARATIVE_NAV_ATTRS], ) - const results: string[] = [] + // Route every harvested href through resolveUrl (resolve-against-page + + // in-scope + normalizeUrl) rather than an ad-hoc inline normalization. This + // aligns DOM-link dedup with the rest of the crawl — e.g. bare "#section" + // scroll anchors collapse to the base page instead of spawning phantom + // targets, while "#/route" hash-router URLs are preserved. const seen = new Set() - + const results: string[] = [] for (const href of hrefs) { - try { - const u = new URL(href) - if (!inScope(u.hostname)) continue - const normalized = u.origin + u.pathname + u.search + u.hash - if (!seen.has(normalized)) { - seen.add(normalized) - results.push(normalized) - } - } catch {} + const resolved = resolveUrl(href, pageUrl, inScope) + if (resolved && !seen.has(resolved)) { + seen.add(resolved) + results.push(resolved) + } } - return results } @@ -2145,7 +2170,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo continue } - // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectDOMLinks + // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectNavLinks const normalizedEntryUrl = normalizeUrl(entry.url) visitedPages.add(normalizedEntryUrl) globalState.visitedPages.add(normalizedEntryUrl) // sync for explorePageWithAI's filterVisitedLinks @@ -2253,7 +2278,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo // Collect DOM links from each context — enqueue with context tag for (const ctx of visitableContexts) { - const domLinks = await collectDOMLinks(ctx.page, entry.url, inScope) + const domLinks = await collectNavLinks(ctx.page, entry.url, inScope) for (const url of domLinks) { enqueueWithContext(url, ctx.id, pageQueue, visitedPages, inScope, pathPatternCounts) } @@ -2632,7 +2657,7 @@ export async function run(config: AgentConfig): Promise { if (newFingerprint === oldFingerprint) { log.info("page unchanged after auth, skipping exploration", { url: currentUrl }) // Still collect DOM links — navbar may have new links after login - const domLinks = await collectDOMLinks(page, currentUrl, inScope) + const domLinks = await collectNavLinks(page, currentUrl, inScope) for (const url of domLinks) { enqueueUrl(url, globalState, inScope) }