Make waitUntilLoaded best-effort: one deadline, no throwing, report each signal (#87)

markanywhere-browse/README.md

@@ -96,10 +96,20 @@ val dump = PageSession(tab).dump()
    never sees).
 
 Steps 2 and 3 run concurrently and **both** must report quiet — each covers the
-other's blind spot. All three return a boolean: `true` if it genuinely settled,
-`false` if a `timeout` cap tripped first. A `false` means "best effort, proceed
-anyway", **not** "failed" — on a never-quiet page you still capture. The waiters
-are also usable standalone (`tab.waitForNetworkIdle()`, `tab.waitForDomIdle()`).
+other's blind spot. All three steps share the one `timeout` budget, so the call
+returns within it however the page behaves, and it never throws on page
+behaviour: an evaluation lost to a navigation (e.g. right after a click) is
+retried against the new document.
+
+`waitUntilLoaded()` returns a `PageLoad` reporting each signal. `settled` is
+`true` if the page genuinely settled; `false` means "best effort, proceed
+anyway", **not** "failed" — on a never-quiet page you still capture. `parsed`
+tells a busy page apart from a truncated one: when it is `false` the document
+itself never finished parsing (a response still streaming, a parser-blocking
+script never delivered), so a capture would be cut off partway through the body.
+The waiters are also usable standalone (`tab.waitForNetworkIdle()`,
+`tab.waitForDomIdle()`), returning `true` if they settled and `false` if their
+`timeout` cap tripped first.
 
 ## Module notes
 

markanywhere-browse/api/markanywhere-browse.api

@@ -1,3 +1,20 @@
+public final class com/xemantic/markanywhere/browse/PageLoad {
+	public fun <init> (Ldev/kdriver/core/tab/ReadyState;ZZ)V
+	public final fun component1 ()Ldev/kdriver/core/tab/ReadyState;
+	public final fun component2 ()Z
+	public final fun component3 ()Z
+	public final fun copy (Ldev/kdriver/core/tab/ReadyState;ZZ)Lcom/xemantic/markanywhere/browse/PageLoad;
+	public static synthetic fun copy$default (Lcom/xemantic/markanywhere/browse/PageLoad;Ldev/kdriver/core/tab/ReadyState;ZZILjava/lang/Object;)Lcom/xemantic/markanywhere/browse/PageLoad;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getDomIdle ()Z
+	public final fun getNetworkIdle ()Z
+	public final fun getParsed ()Z
+	public final fun getReadyState ()Ldev/kdriver/core/tab/ReadyState;
+	public final fun getSettled ()Z
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class com/xemantic/markanywhere/browse/PageSession {
 	public fun <init> (Ldev/kdriver/core/tab/Tab;)V
 	public final fun dump (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;

markanywhere-browse/src/commonMain/kotlin/WaitUntilLoaded.kt

@@ -17,15 +17,53 @@
 package com.xemantic.markanywhere.browse
 
 import dev.kdriver.cdp.domain.network
+import dev.kdriver.core.exceptions.ConnectionClosedException
 import dev.kdriver.core.tab.ReadyState
 import dev.kdriver.core.tab.Tab
 import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.Channel
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.booleanOrNull
+import kotlinx.serialization.json.contentOrNull
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.milliseconds
 import kotlin.time.Duration.Companion.seconds
+import kotlin.time.TimeSource
+
+/**
+ * The outcome of [waitUntilLoaded]: what each of its three signals reported by
+ * the time it returned.
+ *
+ * @property readyState the last `document.readyState` observed, or `null` if it
+ *   could not be read at all within the timeout (e.g. the page kept navigating).
+ * @property networkIdle whether [waitForNetworkIdle] reported quiet.
+ * @property domIdle whether [waitForDomIdle] reported quiet.
+ */
+public data class PageLoad(
+    val readyState: ReadyState?,
+    val networkIdle: Boolean,
+    val domIdle: Boolean,
+) {
+
+    /**
+     * Whether the page genuinely settled: the document completed, and both the
+     * network and the DOM went quiet. `false` means "best effort, proceed
+     * anyway", not "failed" — a never-quiet page is still worth capturing.
+     */
+    val settled: Boolean
+        get() = readyState == ReadyState.COMPLETE && networkIdle && domIdle
+
+    /**
+     * Whether the whole document has been parsed (`interactive` or `complete`).
+     * When `false` the parser is still waiting for the document itself — a
+     * response still streaming, or a parser-blocking `<script src>` never
+     * delivered — so a capture taken now is cut off partway through the body,
+     * unlike an unsettled page that is merely still busy.
+     */
+    val parsed: Boolean
+        get() = readyState == ReadyState.INTERACTIVE || readyState == ReadyState.COMPLETE
+
+}
 
 /**
  * Best-effort "the page has settled" wait for a *structure-agnostic* capture:
@@ -40,22 +78,97 @@ import kotlin.time.Duration.Companion.seconds
  * Steps 2 and 3 run concurrently and BOTH must report quiet — each covers the
  * other's blind spot (network-idle is momentarily true between request-sent
  * and render; DOM-idle is momentarily true while a fetch is in flight but not
- * yet rendered). On a never-quiet page the [timeout] cap inside each waiter
- * trips and capture proceeds anyway.
+ * yet rendered).
+ *
+ * All three steps share the single [timeout] budget: steps 2 and 3 get only
+ * what step 1 left over, so the call returns within [timeout] (plus a CDP
+ * round-trip) however the page behaves.
  *
- * @return `true` if both the network and the DOM went quiet within [timeout],
- *   `false` if either hit its cap (capture should still proceed — `false`
- *   means "best effort", not "failed").
+ * It never throws on page behaviour. Step 1 polls `document.readyState` itself
+ * rather than calling kdriver's [Tab.waitForReadyState], which throws at its
+ * cap and checks that cap only between evaluations — so a hung evaluation (a
+ * main thread blocked by a long script) was bounded only by kdriver's command
+ * timeout. An evaluation that fails because the page navigated under it (its
+ * JavaScript context destroyed, as right after a click that navigates) is
+ * retried against the new document, in every step. Throwing after a navigation
+ * or click that already happened would make a caller retry an action that went
+ * through. Only a closed browser connection, which no retry can outlive, still
+ * propagates.
+ *
+ * @return what each signal reported — see [PageLoad.settled] and, before
+ *   capturing an unsettled page, [PageLoad.parsed].
  */
 public suspend fun Tab.waitUntilLoaded(
     networkIdleTime: Duration = 500.milliseconds,
     domQuietTime: Duration = 500.milliseconds,
     timeout: Duration = 15.seconds,
-): Boolean = coroutineScope {
-    waitForReadyState(ReadyState.COMPLETE, timeout = timeout.inWholeMilliseconds)
-    val networkIdle = async { waitForNetworkIdle(networkIdleTime, timeout) }
-    val domIdle = async { waitForDomIdle(domQuietTime, timeout) }
-    networkIdle.await() and domIdle.await()
+): PageLoad = coroutineScope {
+    val start = TimeSource.Monotonic.markNow()
+    val remaining = { timeout - start.elapsedNow() }
+    var readyState: ReadyState? = null
+    withTimeoutOrNull(timeout) {
+        while (readyState != ReadyState.COMPLETE) {
+            readyState = retryingOnPageError(fallback = readyState) { readReadyState() }
+            if (readyState != ReadyState.COMPLETE) delay(POLL_INTERVAL)
+        }
+    }
+    val networkIdle = async {
+        withinBudget(remaining) { waitForNetworkIdle(networkIdleTime, it) }
+    }
+    val domIdle = async {
+        withinBudget(remaining) { waitForDomIdle(domQuietTime, it) }
+    }
+    PageLoad(readyState, networkIdle.await(), domIdle.await())
+}
+
+private val POLL_INTERVAL = 100.milliseconds
+
+private suspend fun Tab.readReadyState(): ReadyState? {
+    val value = (rawEvaluate("document.readyState") as? JsonPrimitive)?.contentOrNull
+    return ReadyState.entries.firstOrNull { it.name.equals(value, ignoreCase = true) }
+}
+
+/**
+ * Runs a boolean waiter with whatever is [remaining] of the shared budget,
+ * retrying it after a page error (see [retryingOnPageError]) until the budget
+ * runs out. The waiter is also cut off at the budget, so one that hangs on an
+ * unresponsive page (an in-page timer cannot fire on a blocked main thread)
+ * cannot outlive it.
+ */
+private suspend fun withinBudget(
+    remaining: () -> Duration,
+    waiter: suspend (budget: Duration) -> Boolean,
+): Boolean {
+    while (true) {
+        val budget = remaining()
+        if (!budget.isPositive()) return false
+        withTimeoutOrNull(budget) {
+            retryingOnPageError(fallback = null) { waiter(budget) }
+        }?.let { return it }
+        // a page error (or the budget ran out, which the next pass sees):
+        // pause briefly, then retry against the document now in the tab
+        delay(minOf(POLL_INTERVAL, remaining()))
+    }
+}
+
+/**
+ * Runs [block], answering [fallback] instead when it fails because of the page
+ * — a JavaScript context destroyed by a navigation, a CDP command timing out on
+ * a blocked main thread, an evaluation error — so a caller polling a live page
+ * can simply try again. Cancellation (including an enclosing timeout) and a
+ * closed browser connection still propagate.
+ */
+private suspend fun <T> retryingOnPageError(
+    fallback: T,
+    block: suspend () -> T,
+): T = try {
+    block()
+} catch (e: CancellationException) {
+    throw e
+} catch (e: ConnectionClosedException) {
+    throw e
+} catch (_: Exception) {
+    fallback
 }
 
 /**
@@ -76,7 +189,7 @@ public suspend fun Tab.waitUntilLoaded(
  * races across the [Dispatchers.Default] threads the collectors may run on.
  *
  * @return `true` if the network went idle, `false` if [timeout] elapsed first
- *   (matching [Tab.waitForReadyState]'s boolean contract rather than throwing).
+ *   (a boolean contract rather than throwing, unlike [Tab.waitForReadyState]).
  */
 public suspend fun Tab.waitForNetworkIdle(
     idleTime: Duration = 500.milliseconds,

markanywhere-browse/src/commonTest/html/blocked.html

@@ -0,0 +1,31 @@
+<!--
+  ~ Copyright 2026 Kazimierz Pogoda / Xemantic
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!doctype html>
+<html lang="en">
+<head>
+    <title>Blocked</title>
+</head>
+<body>
+<h1>Blocked</h1>
+<!-- The test pauses this request via the CDP Fetch domain and never answers it,
+     standing in for a parser-blocking script the server never delivers: the
+     parser stops here, so readyState stays "loading" and the paragraph below
+     never reaches the DOM. -->
+<script src="blocked.js"></script>
+<p>After the script</p>
+</body>
+</html>

markanywhere-browse/src/commonTest/html/navigating.html

@@ -0,0 +1,29 @@
+<!--
+  ~ Copyright 2026 Kazimierz Pogoda / Xemantic
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!doctype html>
+<html lang="en">
+<head>
+    <title>Navigating</title>
+</head>
+<body>
+<h1>Navigating</h1>
+<!-- Reloads itself forever, standing in for a click that navigates while the
+     page-settle wait is polling: every reload destroys the JavaScript context
+     an in-flight evaluation runs in. -->
+<script>setTimeout(() => location.reload(), 50);</script>
+</body>
+</html>

markanywhere-browse/src/commonTest/html/stalled.html

@@ -0,0 +1,30 @@
+<!--
+  ~ Copyright 2026 Kazimierz Pogoda / Xemantic
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!doctype html>
+<html lang="en">
+<head>
+    <title>Stalled</title>
+</head>
+<body>
+<h1>Stalled</h1>
+<!-- The test pauses this request via the CDP Fetch domain and never answers it,
+     standing in for a server that never responds (e.g. a Wayback Machine toolbar
+     asset): the document has fully arrived, but readyState never reaches
+     "complete" because the load event waits for this image. -->
+<img src="stalled.png" alt="">
+</body>
+</html>