v3.0.0 chore: sync released main into 3.0.x

README.md

@@ -1,4 +1,4 @@
-> **This is the `3.0.x` release branch: version 3.0.0 of the Rust PRISM server
+> **Version 3.0.0: the Rust PRISM server
 > ([#244](https://github.com/Qbit-Org/qbit-mining-bootstrap/pull/244)).**
 > Deploy it from the `v3.0.0` tag, and upgrade from `2.x.x` only as the
 > [3.0.0 release notes](doc/release-notes-3.0.0.md#upgrading) describe: native

crates/qbit-prism-server/src/coordinator/miner_submit.rs

@@ -1010,23 +1010,43 @@ impl Coordinator {
         (capped, log_path): (BlockAckPath, &'static str),
     ) -> SaveOutcome {
         loop {
+            // A bound already past, as for a capture whose append committed
+            // in the share grace (#657), starts no poll: its answer would
+            // arrive too late to use.
+            if tokio::time::Instant::now() >= bound {
+                break;
+            }
             // Observe credit and disposition in one MVCC snapshot so
             // finalization cannot fall between two separate reads. The credit
             // this polls for is written by the reconciliation of the block
             // this call is still acknowledging, so it lands in the newest
             // leaves and carries the same probe floor as the probe above:
             // without it, every poll of this loop would descend one per-leaf
             // `share_id` index per attached partition.
-            let poll = tokio::time::timeout_at(bound, async {
-                sqlx::query_as::<_, (bool, Option<String>, Option<String>, Option<String>)>(
-                    "SELECT EXISTS(SELECT 1 FROM qbit_share_ledger WHERE share_id=$1 AND share_seq>=qbit_prism_share_probe_floor()), (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT last_error FROM qbit_block_candidate_outbox WHERE block_hash=$2)",
-                )
-                .bind(share_id)
-                .bind(block_hash)
-                .fetch_one(&mut *self.ledger.acquire().await?)
+            //
+            // Each poll runs in its own task, as the probe and enqueue do, and
+            // only the wait for it is bounded. A bound that fell inside the
+            // poll's checkout, during the pool's probe of an idle connection
+            // or a new connection's setup, would drop that connection with
+            // its reply unread: the server sees a reset and the pool loses a
+            // healthy connection. Past the bound the poll finishes on its own
+            // and returns its connection.
+            let poll = tokio::spawn({
+                let ledger = self.ledger.clone();
+                let (share_id, block_hash) = (share_id.to_owned(), block_hash.to_owned());
+                async move {
+                    sqlx::query_as::<_, (bool, Option<String>, Option<String>, Option<String>)>(
+                        "SELECT EXISTS(SELECT 1 FROM qbit_share_ledger WHERE share_id=$1 AND share_seq>=qbit_prism_share_probe_floor()), (SELECT state FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT offer_outcome FROM qbit_block_candidate_outbox WHERE block_hash=$2), (SELECT last_error FROM qbit_block_candidate_outbox WHERE block_hash=$2)",
+                    )
+                    .bind(share_id)
+                    .bind(block_hash)
+                    .fetch_one(&mut *ledger.acquire().await?)
+                    .await
+                }
+            });
+            let poll = tokio::time::timeout_at(bound, poll)
                 .await
-            })
-            .await;
+                .map(|joined| anyhow::Ok(joined??));
             match poll {
                 Err(_) => break,
                 Ok(Ok((true, _, _, _))) => return SaveOutcome::Accepted,

crates/qbit-prism-server/src/coordinator/miner_submit_acquire_tests.rs

@@ -535,18 +535,20 @@ async fn block_only_poll_checkout_cancel_and_deadline_stay_unknown() -> Result<(
             let mut persist = Box::pin(h.persist(start));
             let held = first_poll(h, &mut persist).await?;
             let before = counts(h.metrics());
-            let sum = sample(h.metrics(), "failure", "sum");
             tokio::time::pause();
             let resume = ResumeClock;
             tokio::time::advance(Duration::from_millis(50)).await;
             assert!(futures_util::poll!(&mut persist).is_pending());
             assert_eq!(counts(h.metrics()), before);
             tokio::time::advance(Duration::from_millis(75)).await;
             drop(persist);
-            assert_eq!(counts(h.metrics()), (before.0, before.1 + 1.));
-            assert!((sample(h.metrics(), "failure", "sum") - sum - 0.075).abs() < 0.000001);
+            // The poll runs in a task the submission does not own, so a
+            // cancelled submission leaves its checkout waiting, not failed,
+            // and the poll still completes.
+            assert_eq!(counts(h.metrics()), before);
             drop(resume);
             drop(held);
+            wait_counts(h.metrics(), (before.0 + 1., before.1)).await?;
             let deferred: Value = sqlx::query_scalar(
                 "SELECT share FROM qbit_prism_deferred_shares WHERE block_hash=$1",
             )
@@ -584,9 +586,12 @@ async fn block_only_poll_checkout_cancel_and_deadline_stay_unknown() -> Result<(
                     ..
                 }
             ));
-            assert_eq!(counts(h.metrics()), (before.0, before.1 + 1.));
+            // The bound ends only the wait: the poll's checkout is not
+            // cancelled with it, and completes once the slot is free.
+            assert_eq!(counts(h.metrics()), before);
             drop(resume);
             drop(held);
+            wait_counts(h.metrics(), (before.0 + 1., before.1)).await?;
             Ok(())
         })
     })
@@ -721,9 +726,12 @@ async fn block_only_poll_missing_and_sql_error_stay_unknown() -> Result<()> {
                     panic!("nonterminal: {answer:?}")
                 };
                 assert_eq!(phase, expected_phase);
-                assert_eq!(counts(h.metrics()), (before.0 + 1., before.1 + 1.));
+                // The poll still waiting for the slot at the bound is not
+                // cancelled with the answer; it checks out once released.
+                assert_eq!(counts(h.metrics()), (before.0 + 1., before.1));
                 drop(resume);
                 drop(held);
+                wait_counts(h.metrics(), (before.0 + 2., before.1)).await?;
                 drop(tokio::time::timeout(WAIT, h.ledger().pool.acquire()).await??);
                 Ok(())
             })

docs/prism-pool-acquire-metrics.md

@@ -116,16 +116,18 @@ unchanged.
 The coordinator database clock and chain-observation state each record one
 checkout. The latter retains its one-statement revision/epoch/tip snapshot.
 Each block-only duplicate probe and each credit/disposition poll records its
-own checkout, inside the
-original acknowledgement deadline. The two block-only probes are not issued-job
-reads. Credit and disposition still share one SQL statement and MVCC snapshot.
-Connections are released at the statement boundary, before enqueue or poll
-sleep. Missing rows, duplicates and terminal dispositions retain successful
-checkout observations; SQL errors or cancellation after acquisition do too.
-Cancellation while acquiring records a failure. Before enqueue a read failure
-is definite; after enqueue a failed or missing read does not prove whether the
-durable candidate earned credit. Timing changes neither that distinction nor
-the enqueue's existing handling of an unknown outcome.
+own checkout. Each runs in a task the acknowledgement does not own (#574), so
+neither the acknowledgement deadline nor a cancelled submission cancels its
+checkout or SQL: a checkout still waiting then records its own outcome when it
+ends, and a poll that answers after the deadline is discarded. The two
+block-only probes are not issued-job reads. Credit and disposition still share
+one SQL statement and MVCC snapshot. Connections are released at the statement
+boundary, before enqueue or poll sleep. Missing rows, duplicates and terminal
+dispositions retain successful checkout observations; SQL errors do too.
+Before enqueue a read failure is definite; after enqueue a failed or missing
+read does not prove whether the durable candidate earned credit. Timing changes
+neither that distinction nor the enqueue's existing handling of an unknown
+outcome.
 
 The server passes its existing metrics handle to the rollup loop. Each valid
 rollup attempt records one checkout before `BEGIN`; `SET LOCAL`, the single

docs/releasing.md

@@ -0,0 +1,112 @@
+# Publishing a release
+
+`main` is the release promotion branch. `3.x.x` is the development branch and
+production line for the 3.x series, whose current release is 3.1.0; `2.x.x` is
+the maintenance branch of the 2.x series. A release cut from an earlier point
+of a development branch gets its own branch at that cutoff, as `3.0.x` does
+for 3.0.0. A version is published only when its tag and GitHub release exist.
+A version bump or merged preparation PR alone does not publish it.
+
+## Prepare one release candidate
+
+Keep one pending release version across `VERSION`, `[workspace.package].version`
+in the root `Cargo.toml` (which every crate inherits), both `Cargo.lock` files
+(the root and `crates/qbit-prism-server/fuzz`), and
+`doc/release-notes-<version>.md`. `scripts/check_version_skew.py` checks the
+first three agree. Ordinary development PRs contribute to
+that pending release's notes; they do not each create a patch release.
+Choose the next version after reconciling the last published release and
+any identifiers already distributed to operators.
+
+For the first 2.x release, PR #246 is the single **2.0.0** promotion into
+`main`. It includes the changes temporarily labelled 2.0.1 and 2.0.2 on
+`2.x.x`; neither identifier was tagged or published as a GitHub release.
+Their changes and migration requirements are consolidated in
+[the 2.0.0 notes](../doc/release-notes-2.0.0.md). Do not create retrospective
+2.0.1 or 2.0.2 tags for those preparation commits.
+
+For 3.0.0, the cutoff is `4c134913` on `3.x.x`, the last commit before the
+3.1 dual writer. `3.0.x` starts there and takes the version bump, the 3.0.0
+notes and the release's dependency fixes (#776, merged as `aabe470f`). Unlike
+2.0.0, `v3.0.0` was tagged and published on `3.0.x` at `aabe470f`, before the
+promotion. The promotion into `main` (#778) then gives `main` the tagged tree,
+plus this guide and a README banner that doesn't name a branch; it adds no
+tag. The 3.0.0-rc.N pre-releases were tagged on `3.x.x`. 3.0.0-rc.7 was
+prepared (#772) but never tagged, and its changes ship in 3.0.0. The L3
+matrix for the release is recorded in #779.
+
+3.1.0 needed no release branch: `v3.1.0` was tagged and published on `3.x.x`
+itself, at the squash of its version bump and notes (#782, `59194d03`), as
+`v3.1.0-rc.1` was. The promotion into `main` gives `main` that tagged tree,
+plus this guide, a README banner that names the version, and the released
+3.0.0 notes in place of `3.x.x`'s pre-release copy; the links in the
+development-line note, the migration guide and the ledger operations guide,
+which pointed at the notes at the `v3.0.0` tag, point at that local copy. It
+also keeps #781's disposition-poll change: 3.1.0 made #781's tokio update
+itself (#782) but not the poll change, since it changed no runtime code after
+its release candidate. It adds no tag. #779 records the 400k two-frontend L3
+failures on 3.0.0's tree; 3.1.0 doesn't fix their cause, #716.
+
+Merge the intended development cutoff into the promotion branch and retain
+fixes already made during promotion review. The repository allows only squash
+and rebase merges, so the promotion lands on `main` as one squashed commit. Check package versions together,
+run Docker lint and the relevant test gates, and review upgrade and rollback
+instructions. Leave the release date out while publication is unscheduled;
+set the actual release date in a reviewed commit before the final merge.
+
+## Publish the reviewed main commit
+
+Tag the promoted `main` commit unless the release was already tagged on its
+release or development branch, as 3.0.0 and 3.1.0 were. After the promotion PR
+passes required checks and is merged, identify the exact resulting commit on
+`main`. A squash or rebase merge produces a different commit from the
+promotion branch head. Verify the resulting commit's checks, version metadata
+and dated release notes before tagging it.
+
+After confirming that the tag doesn't already exist, use the reviewed main
+commit in place of `<release-main-sha>`, and the version in place of
+`<version>` (as `v2.0.0` was; `v3.0.0` and `v3.1.0` are signed annotated
+tags on their branches' commits, with their own messages):
+
+```sh
+git fetch origin main --tags
+git tag -s v<version> <release-main-sha> -m 'qbit-mining-bootstrap <version>'
+git verify-tag v<version>
+git push origin refs/tags/v<version>
+git show v<version>:doc/release-notes-<version>.md > /tmp/qbit-release-notes.md
+gh release create v<version> --verify-tag --title 'qbit-mining-bootstrap <version>' \
+  --notes-file /tmp/qbit-release-notes.md
+```
+
+Pushing a `v*` tag also runs the L3 production-window matrix
+(`prism-load-l3.yml`), which promotes the promotion PR's run on the same tree
+or runs `l3-full` (#608).
+
+Run publication commands only when authorized to publish. If a tag or release
+already exists, inspect its target and contents instead of replacing it.
+Use notes from the tagged tree, then verify the GitHub release points to that
+same tag. Keep the upstream qbit binary pin separate from this repository's
+release version.
+
+## Reconcile the development branch
+
+After promotion, open a synchronization PR from a branch that merges the
+released `main` commit into the latest development branch (`3.x.x` after a 3.x
+release, `2.x.x` after 2.0.0), and into a release branch such as `3.0.x` that
+will take patch releases. Preserve subsequent development and resolve metadata
+and notes deliberately; do not reset or force-push the shared branch. Run its
+checks before merging.
+
+For 3.1.0 and 3.0.0 this step merges the released `main` (`b9187551`, #783)
+into `3.x.x`, and `main` at `b9579c67` into `3.0.x`. That commit is 3.0.0's
+promotion (#778) plus #781, before 3.1.0's, because a 3.0 patch branch must
+not take the 3.1 dual writer. Both branches gain this guide, the README banner
+and #781's disposition-poll change. `3.0.x` also gains #781's tokio update,
+and `3.x.x` the released 3.0.0 notes in place of its pre-release copy. The
+next release from either branch must not drop them.
+
+For 2.0.0 this step is required: PR #246 contains promotion fixes and the
+consolidated metadata that were not yet on `2.x.x`. Until this synchronization
+lands, the old 2.0.2 metadata on that branch is not a published release.
+Bring those fixes and the 2.0.0 release record back before selecting the next
+version. Start the next pending notes separately and retain published notes.

scripts/prism_pool_acquires.json

@@ -142,7 +142,7 @@
     "coordinator.rs::submit_loop_with#1::attempt .run(& self , lease)": [1,"non-database"],
     "coordinator.rs::with_tracked_heartbeat#1::self . ledger .acquire()": [2,"ledger"],
     "coordinator/issued_batcher.rs::with_dwell#1::run(ledger , receiver , shutdown . clone ( ) , dwell)": [1,"non-database"],
-    "coordinator/miner_submit.rs::deferred_share_disposition#1::self . ledger .acquire()": [1,"ledger"],
+    "coordinator/miner_submit.rs::deferred_share_disposition#1::ledger .acquire()": [1,"ledger"],
     "coordinator/miner_submit.rs::persist_block_only#1::ledger .acquire()": [1,"ledger"],
     "coordinator/refresh_window.rs::capture_refresh_window#1::prepare(std :: mem :: take ( & mut snapshot . snapshot . prior_balances ) , & admission ,)": [1,"non-database"],
     "coordinator/work_ledger.rs::clocked_payout_revision#1::self .acquire()": [1,"ledger"],