From 5eaf5af90f9f3d8100e31da84ab4ffc4efcda1a9 Mon Sep 17 00:00:00 2001 From: orhanyildirim Date: Sat, 29 Aug 2026 16:13:02 -0400 Subject: [PATCH 1/3] refactor(hackbrowser): route DOM-link harvest through resolveUrl collectDOMLinks normalized hrefs inline (origin+pathname+search+hash) and scoped by hostname, diverging from the rest of the crawl which uses resolveUrl (resolve-against-page + isInScope + normalizeUrl). Route the harvested hrefs through resolveUrl instead: bare "#section" scroll anchors now collapse to the base page (no phantom targets) while "#/route" hash-router URLs are preserved, and dedup matches the BFS queue's keys. Prep for harvesting declarative nav attributes (relative/hash values that must resolve against the page URL). Refs #120 --- packages/hackbrowser/src/agent.ts | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/hackbrowser/src/agent.ts b/packages/hackbrowser/src/agent.ts index b32be831e..20355ab8f 100644 --- a/packages/hackbrowser/src/agent.ts +++ b/packages/hackbrowser/src/agent.ts @@ -1749,21 +1749,20 @@ async function collectDOMLinks(page: Page, pageUrl: string, inScope: ScopeMatche els.map((el) => (el as HTMLAnchorElement).href).filter(Boolean), ) - const results: string[] = [] + // Route every harvested href through resolveUrl (resolve-against-page + + // in-scope + normalizeUrl) rather than an ad-hoc inline normalization. This + // aligns DOM-link dedup with the rest of the crawl — e.g. bare "#section" + // scroll anchors collapse to the base page instead of spawning phantom + // targets, while "#/route" hash-router URLs are preserved. const seen = new Set() - + const results: string[] = [] for (const href of hrefs) { - try { - const u = new URL(href) - if (!inScope(u.hostname)) continue - const normalized = u.origin + u.pathname + u.search + u.hash - if (!seen.has(normalized)) { - seen.add(normalized) - results.push(normalized) - } - } catch {} + const resolved = resolveUrl(href, pageUrl, inScope) + if (resolved && !seen.has(resolved)) { + seen.add(resolved) + results.push(resolved) + } } - return results } From f212be77893e7a99c073fad1577c6680c8c79087 Mon Sep 17 00:00:00 2001 From: orhanyildirim Date: Sat, 29 Aug 2026 16:14:14 -0400 Subject: [PATCH 2/3] feat(hackbrowser): harvest declarative nav attributes beyond MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Button/router-nav SPAs often expose no links, so the DOM-link supplement found nothing and route discovery fell entirely on the planner's per-page clicks — shallow crawls (#120). Widen the harvest to elements that declare a destination without an anchor: data-href, data-url, and role=link. The first destination attribute per element is resolved against the page URL (so relative and #/hash-route values work) and scoped/deduped as before. routerLink/[to]/data-route are excluded on purpose — their values are router-relative and would need scheme-guessing; those routes are discovered imperatively by clicking (phase B). Refs #120 --- packages/hackbrowser/src/agent.ts | 32 ++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/hackbrowser/src/agent.ts b/packages/hackbrowser/src/agent.ts index 20355ab8f..c1262380d 100644 --- a/packages/hackbrowser/src/agent.ts +++ b/packages/hackbrowser/src/agent.ts @@ -1743,10 +1743,36 @@ function resolveUrl(href: string, baseUrl: string, inScope: ScopeMatcher): strin return null } -/** Collect links from DOM as BFS supplement. */ +// Attributes that DECLARATIVELY encode a navigation destination readable +// without clicking (a real URL, path, or hash-route). Order = priority per +// element. `routerLink`/`[to]`/`data-route` are intentionally excluded — their +// values are router-relative and need scheme-guessing; those routes are +// discovered imperatively by clicking (phase B, #120). +const DECLARATIVE_NAV_ATTRS = ["href", "data-href", "data-url"] as const +const NAV_TARGET_SELECTOR = "a[href], [role=link], [data-href], [data-url]" + +/** Collect declarative navigation links from DOM as a BFS supplement. */ async function collectDOMLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise { - const hrefs: string[] = await page.$$eval("a[href]", (els) => - els.map((el) => (el as HTMLAnchorElement).href).filter(Boolean), + // Beyond : many SPAs navigate via non-anchor elements that still + // declare their destination in an attribute (data-href/data-url) or carry + // role=link. Harvest the first destination-bearing attribute per element; the + // raw value is resolved against the page URL below (so "/x" and "#/x" work). + const hrefs: string[] = await page.$$eval( + NAV_TARGET_SELECTOR, + (els, attrs) => { + const out: string[] = [] + for (const el of els) { + for (const attr of attrs) { + const value = el.getAttribute(attr) + if (value) { + out.push(value) + break + } + } + } + return out + }, + [...DECLARATIVE_NAV_ATTRS], ) // Route every harvested href through resolveUrl (resolve-against-page + From ebbcd205d9d5edecbcf89861dd48f7768e0beab5 Mon Sep 17 00:00:00 2001 From: orhanyildirim Date: Sat, 29 Aug 2026 16:15:02 -0400 Subject: [PATCH 3/3] refactor(hackbrowser): rename collectDOMLinks -> collectNavLinks The harvest is no longer -only (it now covers data-href/data-url/ role=link), so the DOM-specific name is misleading. Rename to collectNavLinks across the definition, call sites, and comments. Refs #120 --- packages/hackbrowser/src/agent.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/hackbrowser/src/agent.ts b/packages/hackbrowser/src/agent.ts index c1262380d..86e3c9d4d 100644 --- a/packages/hackbrowser/src/agent.ts +++ b/packages/hackbrowser/src/agent.ts @@ -198,7 +198,7 @@ function waitForBrowserClose(browser: import("playwright").Browser, signal?: Abo /** * Mark that login was detected. The actual re-queue happens in the BFS loop * AFTER the current page finishes exploration (so new discoveries from - * collectDOMLinks are enqueued first, before re-visit URLs). + * collectNavLinks are enqueued first, before re-visit URLs). */ function triggerReDiscovery(globalState: ReturnType): void { if (globalState.authPhase === "authenticated") return @@ -1084,12 +1084,12 @@ async function explorePageWithAI( // 6. Collect same-host links from DOM (BFS supplement) try { - const domLinks = await collectDOMLinks(page, pageUrl, inScope) + const domLinks = await collectNavLinks(page, pageUrl, inScope) for (const url of domLinks) { if (!linksToEnqueue.includes(url)) linksToEnqueue.push(url) } } catch { - log.debug("collectDOMLinks failed (page may have navigated)") + log.debug("collectNavLinks failed (page may have navigated)") } return linksToEnqueue @@ -1752,7 +1752,7 @@ const DECLARATIVE_NAV_ATTRS = ["href", "data-href", "data-url"] as const const NAV_TARGET_SELECTOR = "a[href], [role=link], [data-href], [data-url]" /** Collect declarative navigation links from DOM as a BFS supplement. */ -async function collectDOMLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise { +async function collectNavLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise { // Beyond : many SPAs navigate via non-anchor elements that still // declare their destination in an attribute (data-href/data-url) or carry // role=link. Harvest the first destination-bearing attribute per element; the @@ -2170,7 +2170,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo continue } - // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectDOMLinks + // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectNavLinks const normalizedEntryUrl = normalizeUrl(entry.url) visitedPages.add(normalizedEntryUrl) globalState.visitedPages.add(normalizedEntryUrl) // sync for explorePageWithAI's filterVisitedLinks @@ -2278,7 +2278,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo // Collect DOM links from each context — enqueue with context tag for (const ctx of visitableContexts) { - const domLinks = await collectDOMLinks(ctx.page, entry.url, inScope) + const domLinks = await collectNavLinks(ctx.page, entry.url, inScope) for (const url of domLinks) { enqueueWithContext(url, ctx.id, pageQueue, visitedPages, inScope, pathPatternCounts) } @@ -2657,7 +2657,7 @@ export async function run(config: AgentConfig): Promise { if (newFingerprint === oldFingerprint) { log.info("page unchanged after auth, skipping exploration", { url: currentUrl }) // Still collect DOM links — navbar may have new links after login - const domLinks = await collectDOMLinks(page, currentUrl, inScope) + const domLinks = await collectNavLinks(page, currentUrl, inScope) for (const url of domLinks) { enqueueUrl(url, globalState, inScope) }