diff --git a/markanywhere-browse/README.md b/markanywhere-browse/README.md index 45cbcca..aa8f904 100644 --- a/markanywhere-browse/README.md +++ b/markanywhere-browse/README.md @@ -96,10 +96,20 @@ val dump = PageSession(tab).dump() never sees). Steps 2 and 3 run concurrently and **both** must report quiet — each covers the -other's blind spot. All three return a boolean: `true` if it genuinely settled, -`false` if a `timeout` cap tripped first. A `false` means "best effort, proceed -anyway", **not** "failed" — on a never-quiet page you still capture. The waiters -are also usable standalone (`tab.waitForNetworkIdle()`, `tab.waitForDomIdle()`). +other's blind spot. All three steps share the one `timeout` budget, so the call +returns within it however the page behaves, and it never throws on page +behaviour: an evaluation lost to a navigation (e.g. right after a click) is +retried against the new document. + +`waitUntilLoaded()` returns a `PageLoad` reporting each signal. `settled` is +`true` if the page genuinely settled; `false` means "best effort, proceed +anyway", **not** "failed" — on a never-quiet page you still capture. `parsed` +tells a busy page apart from a truncated one: when it is `false` the document +itself never finished parsing (a response still streaming, a parser-blocking +script never delivered), so a capture would be cut off partway through the body. +The waiters are also usable standalone (`tab.waitForNetworkIdle()`, +`tab.waitForDomIdle()`), returning `true` if they settled and `false` if their +`timeout` cap tripped first. ## Module notes diff --git a/markanywhere-browse/api/markanywhere-browse.api b/markanywhere-browse/api/markanywhere-browse.api index ac91982..761e2ca 100644 --- a/markanywhere-browse/api/markanywhere-browse.api +++ b/markanywhere-browse/api/markanywhere-browse.api @@ -1,3 +1,20 @@ +public final class com/xemantic/markanywhere/browse/PageLoad { + public fun (Ldev/kdriver/core/tab/ReadyState;ZZ)V + public final fun component1 ()Ldev/kdriver/core/tab/ReadyState; + public final fun component2 ()Z + public final fun component3 ()Z + public final fun copy (Ldev/kdriver/core/tab/ReadyState;ZZ)Lcom/xemantic/markanywhere/browse/PageLoad; + public static synthetic fun copy$default (Lcom/xemantic/markanywhere/browse/PageLoad;Ldev/kdriver/core/tab/ReadyState;ZZILjava/lang/Object;)Lcom/xemantic/markanywhere/browse/PageLoad; + public fun equals (Ljava/lang/Object;)Z + public final fun getDomIdle ()Z + public final fun getNetworkIdle ()Z + public final fun getParsed ()Z + public final fun getReadyState ()Ldev/kdriver/core/tab/ReadyState; + public final fun getSettled ()Z + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class com/xemantic/markanywhere/browse/PageSession { public fun (Ldev/kdriver/core/tab/Tab;)V public final fun dump (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt index afcc8fa..ba37fe6 100644 --- a/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt +++ b/markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt @@ -17,15 +17,53 @@ package com.xemantic.markanywhere.browse import dev.kdriver.cdp.domain.network +import dev.kdriver.core.exceptions.ConnectionClosedException import dev.kdriver.core.tab.ReadyState import dev.kdriver.core.tab.Tab import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.contentOrNull import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource + +/** + * The outcome of [waitUntilLoaded]: what each of its three signals reported by + * the time it returned. + * + * @property readyState the last `document.readyState` observed, or `null` if it + * could not be read at all within the timeout (e.g. the page kept navigating). + * @property networkIdle whether [waitForNetworkIdle] reported quiet. + * @property domIdle whether [waitForDomIdle] reported quiet. + */ +public data class PageLoad( + val readyState: ReadyState?, + val networkIdle: Boolean, + val domIdle: Boolean, +) { + + /** + * Whether the page genuinely settled: the document completed, and both the + * network and the DOM went quiet. `false` means "best effort, proceed + * anyway", not "failed" — a never-quiet page is still worth capturing. + */ + val settled: Boolean + get() = readyState == ReadyState.COMPLETE && networkIdle && domIdle + + /** + * Whether the whole document has been parsed (`interactive` or `complete`). + * When `false` the parser is still waiting for the document itself — a + * response still streaming, or a parser-blocking ` +

After the script

+ + diff --git a/markanywhere-browse/src/commonTest/html/navigating.html b/markanywhere-browse/src/commonTest/html/navigating.html new file mode 100644 index 0000000..7970ed3 --- /dev/null +++ b/markanywhere-browse/src/commonTest/html/navigating.html @@ -0,0 +1,29 @@ + + + + + + Navigating + + +

Navigating

+ + + + diff --git a/markanywhere-browse/src/commonTest/html/stalled.html b/markanywhere-browse/src/commonTest/html/stalled.html new file mode 100644 index 0000000..7ef4107 --- /dev/null +++ b/markanywhere-browse/src/commonTest/html/stalled.html @@ -0,0 +1,30 @@ + + + + + + Stalled + + +

Stalled

+ + + + diff --git a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt index 890491d..32f1633 100644 --- a/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt +++ b/markanywhere-browse/src/commonTest/kotlin/WaitUntilLoadedTest.kt @@ -17,10 +17,22 @@ package com.xemantic.markanywhere.browse import com.xemantic.kotlin.test.assert +import dev.kdriver.cdp.domain.Fetch +import dev.kdriver.cdp.domain.fetch +import dev.kdriver.core.tab.ReadyState +import dev.kdriver.core.tab.Tab +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import kotlinx.coroutines.test.runTest import kotlin.test.Test import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource +import kotlin.time.measureTime /** * Integration coverage for the page-settle waiters in `WaitUntilLoaded.kt`. @@ -29,8 +41,9 @@ import kotlin.time.Duration.Companion.seconds * over live CDP signals (`network.*` events, an in-page `MutationObserver`), * so the only honest test drives a real headless browser over a `file://` * fixture engineered to either settle or stay busy. Assertions are on the - * boolean contract only (settled vs. hit-the-cap), never on elapsed time, to - * keep them off the wall clock and non-flaky. + * reported outcome (settled vs. hit-the-cap), not on elapsed time, to keep them + * off the wall clock and non-flaky — the one exception is the shared-budget + * test, whose contract *is* the time spent, asserted with a generous margin. * * Each test runs in its own browser via [runInBrowser] (the module's existing * pattern) and on a real dispatcher — see [CreateBrowserTest] for why @@ -114,7 +127,7 @@ class WaitUntilLoadedTest { @Test fun `waitUntilLoaded should report settled on a quiescent page`() = runTest { - val settled = runInBrowser { browser -> + val load = runInBrowser { browser -> // given val tab = browser.get(testPageUrl("simple.html")) @@ -126,13 +139,14 @@ class WaitUntilLoadedTest { ) } - // then — both network and DOM went quiet within the timeout - assert(settled) + // then — the document completed, and both network and DOM went quiet + assert(load == PageLoad(ReadyState.COMPLETE, networkIdle = true, domIdle = true)) + assert(load.settled) } @Test fun `waitUntilLoaded should report best-effort false when the DOM never settles`() = runTest { - val settled = runInBrowser { browser -> + val load = runInBrowser { browser -> // given — DOM mutates forever, so the dom-idle leg can never confirm val tab = browser.get(testPageUrl("dom-busy.html")) @@ -144,8 +158,133 @@ class WaitUntilLoadedTest { ) } - // then — AND of the two legs is false (capture should still proceed) - assert(!settled) + // then — not settled (capture should still proceed), and only the DOM is to blame + assert(load == PageLoad(ReadyState.COMPLETE, networkIdle = true, domIdle = false)) + assert(!load.settled) } + @Test + fun `waitUntilLoaded should report a parsed document whose load never fires`() = runTest { + val (paused, load) = runInBrowser { browser -> + // given — a fully arrived document whose one image is never answered, + // so the load event never fires and readyState stays "interactive" + val tab = browser.get() + tab.withStalledRequests("*stalled.png") { paused -> + tab.get(testPageUrl("stalled.html")) + + // when + paused.isCompleted to tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + } + + // then — the readyState cap folds into the result instead of throwing, + // and the document is reported as parsed, hence safe to capture + assert(paused) + assert(load.readyState == ReadyState.INTERACTIVE) + assert(load.parsed) + assert(!load.settled) + } + + @Test + fun `waitUntilLoaded should report a document the parser never finished`() = runTest { + val (paused, load) = runInBrowser { browser -> + // given — a parser-blocking script the server never delivers, so the + // document is cut off before the rest of its body + val tab = browser.get() + tab.withStalledRequests("*blocked.js") { paused -> + tab.get(testPageUrl("blocked.html")) + + // when + paused.isCompleted to tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + } + + // then — distinguishable from a harmless "still busy": the capture would be truncated + assert(paused) + assert(load.readyState == ReadyState.LOADING) + assert(!load.parsed) + assert(!load.settled) + } + + @Test + fun `waitUntilLoaded should not throw when the page navigates during the wait`() = runTest { + val load = runInBrowser { browser -> + // given — a page reloading itself every 50ms, so evaluations keep + // losing their JavaScript context mid-flight, as after a click that navigates + val tab = browser.get(testPageUrl("navigating.html")) + + // when + tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 1500.milliseconds, + ) + } + + // then — best effort: the page never settled, but the wait still returned + assert(!load.settled) + } + + @Test + fun `waitUntilLoaded should spend a single timeout across all its steps`() = runTest { + val elapsed = runInBrowser { browser -> + // given — the load event never fires (stalled image) AND the DOM never + // goes quiet, so every step runs to its cap + val tab = browser.get() + tab.withStalledRequests("*stalled.png") { + tab.get(testPageUrl("stalled.html")) + tab.rawEvaluate("setInterval(() => document.body.dataset.tick = Date.now(), 50)") + + // when + TimeSource.Monotonic.measureTime { + tab.waitUntilLoaded( + networkIdleTime = 300.milliseconds, + domQuietTime = 300.milliseconds, + timeout = 3.seconds, + ) + } + } + } + + // then — one budget, not one per step (which would take 6s or more); + // the margin absorbs CDP round-trips on a slow machine + assert(elapsed < 4500.milliseconds) + } + +} + +/** + * Pauses every request matching [urlPattern] via the CDP Fetch domain and never + * answers it — a server that never responds — for the duration of [block]. + * + * [block] receives a deferred completed once a request was actually paused, so a + * test can assert the interception happened rather than silently exercising a + * request that failed fast (e.g. a browser not intercepting `file://` URLs). + * Interception is disabled afterwards, which releases the paused request. + */ +private suspend fun Tab.withStalledRequests( + urlPattern: String, + block: suspend (paused: Deferred) -> T, +): T = coroutineScope { + val paused = CompletableDeferred() + // UNDISPATCHED so the collector is subscribed before interception is enabled + val collector = launch(start = CoroutineStart.UNDISPATCHED) { + fetch.requestPaused.first() + paused.complete(Unit) + } + fetch.enable(patterns = listOf(Fetch.RequestPattern(urlPattern = urlPattern))) + try { + block(paused) + } finally { + collector.cancel() + fetch.disable() + } } diff --git a/markanywhere-html/src/jvmTest/kotlin/Capture.kt b/markanywhere-html/src/jvmTest/kotlin/Capture.kt index 44f0c6a..e2cc5a3 100644 --- a/markanywhere-html/src/jvmTest/kotlin/Capture.kt +++ b/markanywhere-html/src/jvmTest/kotlin/Capture.kt @@ -43,7 +43,8 @@ import kotlin.time.Duration.Companion.seconds * font- and viewport-sensitive and should match what a real user sees * - `--wait=` hard cap for the post-navigation settle wait * ([waitUntilLoaded]); capture proceeds once the network and DOM go quiet, - * or this backstop elapses on a never-quiet page (default 15) + * or this backstop elapses on a never-quiet page (default 15) — with a + * warning if the document itself never finished parsing * - `--viewport=x` window size, default `1920x1080` (Full HD). Fixtures * must be captured at a stable **desktop** width: a headless browser's small * default viewport drops below responsive breakpoints, so a CSS navbar @@ -84,7 +85,13 @@ fun main(args: Array) { ) try { val tab = browser.get(url) - tab.waitUntilLoaded(timeout = wait.seconds) + val load = tab.waitUntilLoaded(timeout = wait.seconds) + if (!load.parsed) { + System.err.println( + "warning: the document never finished parsing " + + "(readyState ${load.readyState}), the capture is likely truncated" + ) + } val dump = PageSession(tab).dump() output.writeText(markanywhereJson.encodeToString(dump)) } finally {