From afab1c1902f380d36cea347a6376f90a40f36084 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 30 Sep 2026 16:03:44 +0200 Subject: [PATCH 1/2] Fold waitForReadyState timeout into waitUntilLoaded's best-effort result (#87) kdriver's Tab.waitForReadyState throws TimeoutWaitingForReadyStateException at its cap, so a page whose document fully arrived but whose load event never fires (one sub-resource never answered) made waitUntilLoaded throw instead of returning false. Catch it and report best-effort false, matching the network and DOM waiters. Co-Authored-By: Claude Opus 5.5 --- .../src/commonMain/kotlin/WaitUntilLoaded.kt | 24 +++++++++++---- .../src/commonTest/html/stalled.html | 30 +++++++++++++++++++ .../commonTest/kotlin/WaitUntilLoadedTest.kt | 24 +++++++++++++++ 3 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 markanywhere-browse/src/commonTest/html/stalled.html diff --git a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt index afcc8fa..40ce33d 100644 --- a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt +++ b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt @@ -17,6 +17,7 @@ package com.xemantic.markanywhere.browse import dev.kdriver.cdp.domain.network +import dev.kdriver.core.exceptions.TimeoutWaitingForReadyStateException import dev.kdriver.core.tab.ReadyState import dev.kdriver.core.tab.Tab import kotlinx.coroutines.* @@ -43,19 +44,30 @@ import kotlin.time.Duration.Companion.seconds * yet rendered). On a never-quiet page the [timeout] cap inside each waiter * trips and capture proceeds anyway. * - * @return `true` if both the network and the DOM went quiet within [timeout], - * `false` if either hit its cap (capture should still proceed — `false` - * means "best effort", not "failed"). + * Unlike the other two, kdriver's [Tab.waitForReadyState] *throws* + * [TimeoutWaitingForReadyStateException] at its cap, so step 1 catches it and + * folds it into the result: a page whose document has fully arrived but whose + * `load` never fires (one sub-resource the server never answers) is still + * capturable, and throwing after a navigation or click that already happened + * would make a caller retry an action that went through. + * + * @return `true` if the document completed and both the network and the DOM + * went quiet within [timeout], `false` if any of them hit its cap (capture + * should still proceed — `false` means "best effort", not "failed"). */ public suspend fun Tab.waitUntilLoaded( networkIdleTime: Duration = 500.milliseconds, domQuietTime: Duration = 500.milliseconds, timeout: Duration = 15.seconds, ): Boolean = coroutineScope { - waitForReadyState(ReadyState.COMPLETE, timeout = timeout.inWholeMilliseconds) + val complete = try { + waitForReadyState(ReadyState.COMPLETE, timeout = timeout.inWholeMilliseconds) + } catch (_: TimeoutWaitingForReadyStateException) { + false + } val networkIdle = async { waitForNetworkIdle(networkIdleTime, timeout) } val domIdle = async { waitForDomIdle(domQuietTime, timeout) } - networkIdle.await() and domIdle.await() + complete and networkIdle.await() and domIdle.await() } /** @@ -76,7 +88,7 @@ public suspend fun Tab.waitUntilLoaded( * races across the [Dispatchers.Default] threads the collectors may run on. * * @return `true` if the network went idle, `false` if [timeout] elapsed first - * (matching [Tab.waitForReadyState]'s boolean contract rather than throwing). + * (a boolean contract rather than throwing, unlike [Tab.waitForReadyState]). */ public suspend fun Tab.waitForNetworkIdle( idleTime: Duration = 500.milliseconds, diff --git a/markanywhere-browse/src/commonTest/html/stalled.html b/markanywhere-browse/src/commonTest/html/stalled.html new file mode 100644 index 0000000..7ef4107 --- /dev/null +++ b/markanywhere-browse/src/commonTest/html/stalled.html @@ -0,0 +1,30 @@ + + + + + + Stalled + + +

Stalled

+ + + + diff --git a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt index 890491d..4583dfb 100644 --- a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt +++ b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt @@ -17,6 +17,8 @@ package com.xemantic.markanywhere.browse import com.xemantic.kotlin.test.assert +import dev.kdriver.cdp.domain.Fetch +import dev.kdriver.cdp.domain.fetch import kotlinx.coroutines.test.runTest import kotlin.test.Test import kotlin.time.Duration.Companion.milliseconds @@ -148,4 +150,26 @@ class WaitUntilLoadedTest { assert(!settled) } + @Test + fun `waitUntilLoaded should report best-effort false when readyState never completes`() = runTest { + val settled = runInBrowser { browser -> + // given — a fully arrived document whose one image is never answered + // (paused via the Fetch domain, never continued), so the load event + // never fires and readyState stays "interactive" + val tab = browser.get() + tab.fetch.enable(patterns = listOf(Fetch.RequestPattern(urlPattern = "*stalled.png"))) + tab.get(testPageUrl("stalled.html")) + + // when + tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + + // then — the readyState cap folds into the result instead of throwing + assert(!settled) + } + } From c5fce511170f0e5429f045cdd922e1c9dcf34537 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 30 Sep 2026 16:48:30 +0200 Subject: [PATCH 2/2] Bound waitUntilLoaded by one deadline and report each signal (#87) waitUntilLoaded polls document.readyState itself instead of calling kdriver's waitForReadyState, which checks its cap only between evaluations. All three steps now share the single timeout: the network and DOM waiters get what the readyState poll left, and each is cut off at the deadline, so the call returns within timeout (it took 2x before). A step failing because of the page (a navigation destroying the JS context, a CDP command timeout, an evaluation error) is retried against the new document instead of escaping; only a closed connection throws. It now returns PageLoad (last readyState, networkIdle, domIdle) instead of a Boolean: settled is the old result, and parsed tells a merely busy page apart from a document that never finished parsing, whose capture would be truncated. Capture.kt warns in that case. Tests assert that the Fetch interception actually paused the request and disable it afterwards, and cover a parser-blocked document, a page navigating during the wait, and the shared budget. Co-Authored-By: Claude Opus 5.5 --- markanywhere-browse/README.md | 18 ++- .../api/markanywhere-browse.api | 17 ++ .../src/commonMain/kotlin/WaitUntilLoaded.kt | 141 ++++++++++++++--- .../src/commonTest/html/blocked.html | 31 ++++ .../src/commonTest/html/navigating.html | 29 ++++ .../commonTest/kotlin/WaitUntilLoadedTest.kt | 149 ++++++++++++++++-- .../src/jvmTest/kotlin/Capture.kt | 11 +- 7 files changed, 353 insertions(+), 43 deletions(-) create mode 100644 markanywhere-browse/src/commonTest/html/blocked.html create mode 100644 markanywhere-browse/src/commonTest/html/navigating.html diff --git a/markanywhere-browse/README.md b/markanywhere-browse/README.md index 45cbcca..aa8f904 100644 --- a/markanywhere-browse/README.md +++ b/markanywhere-browse/README.md @@ -96,10 +96,20 @@ val dump = PageSession(tab).dump() never sees). Steps 2 and 3 run concurrently and **both** must report quiet — each covers the -other's blind spot. All three return a boolean: `true` if it genuinely settled, -`false` if a `timeout` cap tripped first. A `false` means "best effort, proceed -anyway", **not** "failed" — on a never-quiet page you still capture. The waiters -are also usable standalone (`tab.waitForNetworkIdle()`, `tab.waitForDomIdle()`). +other's blind spot. All three steps share the one `timeout` budget, so the call +returns within it however the page behaves, and it never throws on page +behaviour: an evaluation lost to a navigation (e.g. right after a click) is +retried against the new document. + +`waitUntilLoaded()` returns a `PageLoad` reporting each signal. `settled` is +`true` if the page genuinely settled; `false` means "best effort, proceed +anyway", **not** "failed" — on a never-quiet page you still capture. `parsed` +tells a busy page apart from a truncated one: when it is `false` the document +itself never finished parsing (a response still streaming, a parser-blocking +script never delivered), so a capture would be cut off partway through the body. +The waiters are also usable standalone (`tab.waitForNetworkIdle()`, +`tab.waitForDomIdle()`), returning `true` if they settled and `false` if their +`timeout` cap tripped first. ## Module notes diff --git a/markanywhere-browse/api/markanywhere-browse.api b/markanywhere-browse/api/markanywhere-browse.api index ac91982..761e2ca 100644 --- a/markanywhere-browse/api/markanywhere-browse.api +++ b/markanywhere-browse/api/markanywhere-browse.api @@ -1,3 +1,20 @@ +public final class com/xemantic/markanywhere/browse/PageLoad { + public fun (Ldev/kdriver/core/tab/ReadyState;ZZ)V + public final fun component1 ()Ldev/kdriver/core/tab/ReadyState; + public final fun component2 ()Z + public final fun component3 ()Z + public final fun copy (Ldev/kdriver/core/tab/ReadyState;ZZ)Lcom/xemantic/markanywhere/browse/PageLoad; + public static synthetic fun copy$default (Lcom/xemantic/markanywhere/browse/PageLoad;Ldev/kdriver/core/tab/ReadyState;ZZILjava/lang/Object;)Lcom/xemantic/markanywhere/browse/PageLoad; + public fun equals (Ljava/lang/Object;)Z + public final fun getDomIdle ()Z + public final fun getNetworkIdle ()Z + public final fun getParsed ()Z + public final fun getReadyState ()Ldev/kdriver/core/tab/ReadyState; + public final fun getSettled ()Z + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class com/xemantic/markanywhere/browse/PageSession { public fun (Ldev/kdriver/core/tab/Tab;)V public final fun dump (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt index 40ce33d..ba37fe6 100644 --- a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt +++ b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt @@ -17,16 +17,53 @@ package com.xemantic.markanywhere.browse import dev.kdriver.cdp.domain.network -import dev.kdriver.core.exceptions.TimeoutWaitingForReadyStateException +import dev.kdriver.core.exceptions.ConnectionClosedException import dev.kdriver.core.tab.ReadyState import dev.kdriver.core.tab.Tab import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.contentOrNull import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource + +/** + * The outcome of [waitUntilLoaded]: what each of its three signals reported by + * the time it returned. + * + * @property readyState the last `document.readyState` observed, or `null` if it + * could not be read at all within the timeout (e.g. the page kept navigating). + * @property networkIdle whether [waitForNetworkIdle] reported quiet. + * @property domIdle whether [waitForDomIdle] reported quiet. + */ +public data class PageLoad( + val readyState: ReadyState?, + val networkIdle: Boolean, + val domIdle: Boolean, +) { + + /** + * Whether the page genuinely settled: the document completed, and both the + * network and the DOM went quiet. `false` means "best effort, proceed + * anyway", not "failed" — a never-quiet page is still worth capturing. + */ + val settled: Boolean + get() = readyState == ReadyState.COMPLETE && networkIdle && domIdle + + /** + * Whether the whole document has been parsed (`interactive` or `complete`). + * When `false` the parser is still waiting for the document itself — a + * response still streaming, or a parser-blocking ` +

After the script

+ + diff --git a/markanywhere-browse/src/commonTest/html/navigating.html b/markanywhere-browse/src/commonTest/html/navigating.html new file mode 100644 index 0000000..7970ed3 --- /dev/null +++ b/markanywhere-browse/src/commonTest/html/navigating.html @@ -0,0 +1,29 @@ + + + + + + Navigating + + +

Navigating

+ + + + diff --git a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt index 4583dfb..32f1633 100644 --- a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt +++ b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt @@ -19,10 +19,20 @@ package com.xemantic.markanywhere.browse import com.xemantic.kotlin.test.assert import dev.kdriver.cdp.domain.Fetch import dev.kdriver.cdp.domain.fetch +import dev.kdriver.core.tab.ReadyState +import dev.kdriver.core.tab.Tab +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import kotlinx.coroutines.test.runTest import kotlin.test.Test import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource +import kotlin.time.measureTime /** * Integration coverage for the page-settle waiters in `WaitUntilLoaded.kt`. @@ -31,8 +41,9 @@ import kotlin.time.Duration.Companion.seconds * over live CDP signals (`network.*` events, an in-page `MutationObserver`), * so the only honest test drives a real headless browser over a `file://` * fixture engineered to either settle or stay busy. Assertions are on the - * boolean contract only (settled vs. hit-the-cap), never on elapsed time, to - * keep them off the wall clock and non-flaky. + * reported outcome (settled vs. hit-the-cap), not on elapsed time, to keep them + * off the wall clock and non-flaky — the one exception is the shared-budget + * test, whose contract *is* the time spent, asserted with a generous margin. * * Each test runs in its own browser via [runInBrowser] (the module's existing * pattern) and on a real dispatcher — see [CreateBrowserTest] for why @@ -116,7 +127,7 @@ class WaitUntilLoadedTest { @Test fun `waitUntilLoaded should report settled on a quiescent page`() = runTest { - val settled = runInBrowser { browser -> + val load = runInBrowser { browser -> // given val tab = browser.get(testPageUrl("simple.html")) @@ -128,13 +139,14 @@ class WaitUntilLoadedTest { ) } - // then — both network and DOM went quiet within the timeout - assert(settled) + // then — the document completed, and both network and DOM went quiet + assert(load == PageLoad(ReadyState.COMPLETE, networkIdle = true, domIdle = true)) + assert(load.settled) } @Test fun `waitUntilLoaded should report best-effort false when the DOM never settles`() = runTest { - val settled = runInBrowser { browser -> + val load = runInBrowser { browser -> // given — DOM mutates forever, so the dom-idle leg can never confirm val tab = browser.get(testPageUrl("dom-busy.html")) @@ -146,19 +158,68 @@ class WaitUntilLoadedTest { ) } - // then — AND of the two legs is false (capture should still proceed) - assert(!settled) + // then — not settled (capture should still proceed), and only the DOM is to blame + assert(load == PageLoad(ReadyState.COMPLETE, networkIdle = true, domIdle = false)) + assert(!load.settled) } @Test - fun `waitUntilLoaded should report best-effort false when readyState never completes`() = runTest { - val settled = runInBrowser { browser -> - // given — a fully arrived document whose one image is never answered - // (paused via the Fetch domain, never continued), so the load event - // never fires and readyState stays "interactive" + fun `waitUntilLoaded should report a parsed document whose load never fires`() = runTest { + val (paused, load) = runInBrowser { browser -> + // given — a fully arrived document whose one image is never answered, + // so the load event never fires and readyState stays "interactive" val tab = browser.get() - tab.fetch.enable(patterns = listOf(Fetch.RequestPattern(urlPattern = "*stalled.png"))) - tab.get(testPageUrl("stalled.html")) + tab.withStalledRequests("*stalled.png") { paused -> + tab.get(testPageUrl("stalled.html")) + + // when + paused.isCompleted to tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + } + + // then — the readyState cap folds into the result instead of throwing, + // and the document is reported as parsed, hence safe to capture + assert(paused) + assert(load.readyState == ReadyState.INTERACTIVE) + assert(load.parsed) + assert(!load.settled) + } + + @Test + fun `waitUntilLoaded should report a document the parser never finished`() = runTest { + val (paused, load) = runInBrowser { browser -> + // given — a parser-blocking script the server never delivers, so the + // document is cut off before the rest of its body + val tab = browser.get() + tab.withStalledRequests("*blocked.js") { paused -> + tab.get(testPageUrl("blocked.html")) + + // when + paused.isCompleted to tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + } + + // then — distinguishable from a harmless "still busy": the capture would be truncated + assert(paused) + assert(load.readyState == ReadyState.LOADING) + assert(!load.parsed) + assert(!load.settled) + } + + @Test + fun `waitUntilLoaded should not throw when the page navigates during the wait`() = runTest { + val load = runInBrowser { browser -> + // given — a page reloading itself every 50ms, so evaluations keep + // losing their JavaScript context mid-flight, as after a click that navigates + val tab = browser.get(testPageUrl("navigating.html")) // when tab.waitUntilLoaded( @@ -168,8 +229,62 @@ class WaitUntilLoadedTest { ) } - // then — the readyState cap folds into the result instead of throwing - assert(!settled) + // then — best effort: the page never settled, but the wait still returned + assert(!load.settled) } + @Test + fun `waitUntilLoaded should spend a single timeout across all its steps`() = runTest { + val elapsed = runInBrowser { browser -> + // given — the load event never fires (stalled image) AND the DOM never + // goes quiet, so every step runs to its cap + val tab = browser.get() + tab.withStalledRequests("*stalled.png") { + tab.get(testPageUrl("stalled.html")) + tab.rawEvaluate("setInterval(() => document.body.dataset.tick = Date.now(), 50)") + + // when + TimeSource.Monotonic.measureTime { + tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 3.seconds, + ) + } + } + } + + // then — one budget, not one per step (which would take 6s or more); + // the margin absorbs CDP round-trips on a slow machine + assert(elapsed < 4500.milliseconds) + } + +} + +/** + * Pauses every request matching [urlPattern] via the CDP Fetch domain and never + * answers it — a server that never responds — for the duration of [block]. + * + * [block] receives a deferred completed once a request was actually paused, so a + * test can assert the interception happened rather than silently exercising a + * request that failed fast (e.g. a browser not intercepting `file://` URLs). + * Interception is disabled afterwards, which releases the paused request. + */ +private suspend fun Tab.withStalledRequests( + urlPattern: String, + block: suspend (paused: Deferred) -> T, +): T = coroutineScope { + val paused = CompletableDeferred() + // UNDISPATCHED so the collector is subscribed before interception is enabled + val collector = launch(start = CoroutineStart.UNDISPATCHED) { + fetch.requestPaused.first() + paused.complete(Unit) + } + fetch.enable(patterns = listOf(Fetch.RequestPattern(urlPattern = urlPattern))) + try { + block(paused) + } finally { + collector.cancel() + fetch.disable() + } } diff --git a/markanywhere-html/src/jvmTest/kotlin/Capture.kt b/markanywhere-html/src/jvmTest/kotlin/Capture.kt index 44f0c6a..e2cc5a3 100644 --- a/markanywhere-html/src/jvmTest/kotlin/Capture.kt +++ b/markanywhere-html/src/jvmTest/kotlin/Capture.kt @@ -43,7 +43,8 @@ import kotlin.time.Duration.Companion.seconds * font- and viewport-sensitive and should match what a real user sees * - `--wait=` hard cap for the post-navigation settle wait * ([waitUntilLoaded]); capture proceeds once the network and DOM go quiet, - * or this backstop elapses on a never-quiet page (default 15) + * or this backstop elapses on a never-quiet page (default 15) — with a + * warning if the document itself never finished parsing * - `--viewport=x` window size, default `1920x1080` (Full HD). Fixtures * must be captured at a stable **desktop** width: a headless browser's small * default viewport drops below responsive breakpoints, so a CSS navbar @@ -84,7 +85,13 @@ fun main(args: Array) { ) try { val tab = browser.get(url) - tab.waitUntilLoaded(timeout = wait.seconds) + val load = tab.waitUntilLoaded(timeout = wait.seconds) + if (!load.parsed) { + System.err.println( + "warning: the document never finished parsing " + + "(readyState ${load.readyState}), the capture is likely truncated" + ) + } val dump = PageSession(tab).dump() output.writeText(markanywhereJson.encodeToString(dump)) } finally {